2024-11-15 22:11:20,684 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-15 22:11:20,699 main DEBUG Took 0.013505 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-15 22:11:20,700 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-15 22:11:20,700 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-15 22:11:20,702 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-15 22:11:20,703 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:11:20,714 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-15 22:11:20,727 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:11:20,728 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:11:20,729 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:11:20,729 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:11:20,730 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:11:20,730 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:11:20,731 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:11:20,731 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:11:20,732 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:11:20,732 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:11:20,733 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:11:20,733 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:11:20,734 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:11:20,734 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:11:20,735 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:11:20,735 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:11:20,736 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:11:20,736 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:11:20,737 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:11:20,737 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:11:20,738 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:11:20,738 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:11:20,739 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:11:20,739 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:11:20,740 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:11:20,740 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-15 22:11:20,742 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:11:20,744 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-15 22:11:20,746 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-15 22:11:20,747 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-15 22:11:20,748 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-15 22:11:20,748 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-15 22:11:20,760 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-15 22:11:20,763 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-15 22:11:20,765 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-15 22:11:20,765 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-15 22:11:20,766 main DEBUG createAppenders(={Console}) 2024-11-15 22:11:20,767 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-15 22:11:20,767 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-15 22:11:20,767 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-15 22:11:20,768 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-15 22:11:20,768 main DEBUG OutputStream closed 2024-11-15 22:11:20,769 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-15 22:11:20,769 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-15 22:11:20,770 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-15 22:11:20,861 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-15 22:11:20,863 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-15 22:11:20,865 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-15 22:11:20,866 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-15 22:11:20,866 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-15 22:11:20,867 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-15 22:11:20,867 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-15 22:11:20,868 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-15 22:11:20,868 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-15 22:11:20,869 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-15 22:11:20,869 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-15 22:11:20,869 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-15 22:11:20,870 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-15 22:11:20,870 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-15 22:11:20,871 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-15 22:11:20,871 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-15 22:11:20,871 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-15 22:11:20,872 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-15 22:11:20,875 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15 22:11:20,876 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-15 22:11:20,876 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-15 22:11:20,877 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-15T22:11:21,219 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf 2024-11-15 22:11:21,224 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-15 22:11:21,225 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15T22:11:21,239 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-15T22:11:21,274 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-15T22:11:21,308 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/cluster_048f81ed-d9d5-ea35-fb33-ebaa9399bbca, deleteOnExit=true 2024-11-15T22:11:21,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-15T22:11:21,310 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/test.cache.data in system properties and HBase conf 2024-11-15T22:11:21,311 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/hadoop.tmp.dir in system properties and HBase conf 2024-11-15T22:11:21,311 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/hadoop.log.dir in system properties and HBase conf 2024-11-15T22:11:21,312 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-15T22:11:21,314 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-15T22:11:21,314 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-15T22:11:21,458 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-15T22:11:21,581 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T22:11:21,586 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-15T22:11:21,587 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-15T22:11:21,587 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-15T22:11:21,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T22:11:21,589 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-15T22:11:21,589 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-15T22:11:21,589 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T22:11:21,590 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T22:11:21,590 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-15T22:11:21,591 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/nfs.dump.dir in system properties and HBase conf 2024-11-15T22:11:21,591 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/java.io.tmpdir in system properties and HBase conf 2024-11-15T22:11:21,592 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T22:11:21,592 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-15T22:11:21,592 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-15T22:11:22,560 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-15T22:11:22,661 INFO [Time-limited test {}] log.Log(170): Logging initialized @2956ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-15T22:11:22,760 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T22:11:22,841 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T22:11:22,868 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T22:11:22,869 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T22:11:22,871 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-15T22:11:22,889 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T22:11:22,895 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/hadoop.log.dir/,AVAILABLE} 2024-11-15T22:11:22,896 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T22:11:23,148 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@322e478f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/java.io.tmpdir/jetty-localhost-46323-hadoop-hdfs-3_4_1-tests_jar-_-any-7760863419620437279/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T22:11:23,157 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:46323} 2024-11-15T22:11:23,157 INFO [Time-limited test {}] server.Server(415): Started @3453ms 2024-11-15T22:11:23,669 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T22:11:23,675 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T22:11:23,677 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T22:11:23,677 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T22:11:23,678 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-15T22:11:23,678 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@288b0180{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/hadoop.log.dir/,AVAILABLE} 2024-11-15T22:11:23,679 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3de8243e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T22:11:23,801 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1ebb9f9f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/java.io.tmpdir/jetty-localhost-35363-hadoop-hdfs-3_4_1-tests_jar-_-any-13273951123278333529/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T22:11:23,801 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@bd2e4d1{HTTP/1.1, (http/1.1)}{localhost:35363} 2024-11-15T22:11:23,802 INFO [Time-limited test {}] server.Server(415): Started @4097ms 2024-11-15T22:11:23,856 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T22:11:24,413 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/cluster_048f81ed-d9d5-ea35-fb33-ebaa9399bbca/data/data1/current/BP-8675891-172.17.0.3-1731708682292/current, will proceed with Du for space computation calculation, 2024-11-15T22:11:24,414 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/cluster_048f81ed-d9d5-ea35-fb33-ebaa9399bbca/data/data2/current/BP-8675891-172.17.0.3-1731708682292/current, will proceed with Du for space computation calculation, 2024-11-15T22:11:24,482 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T22:11:24,542 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xca78b386d00ea7ed with lease ID 0x56c7673d4851ad62: Processing first storage report for DS-c935f7a9-6b24-44fb-a793-fac06a5ecde6 from datanode DatanodeRegistration(127.0.0.1:35039, datanodeUuid=723faa33-b9ca-4c30-bbb3-1907a5afaa48, infoPort=34651, infoSecurePort=0, ipcPort=40991, storageInfo=lv=-57;cid=testClusterID;nsid=1181013597;c=1731708682292) 2024-11-15T22:11:24,543 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xca78b386d00ea7ed with lease ID 0x56c7673d4851ad62: from storage DS-c935f7a9-6b24-44fb-a793-fac06a5ecde6 node DatanodeRegistration(127.0.0.1:35039, datanodeUuid=723faa33-b9ca-4c30-bbb3-1907a5afaa48, infoPort=34651, infoSecurePort=0, ipcPort=40991, storageInfo=lv=-57;cid=testClusterID;nsid=1181013597;c=1731708682292), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-15T22:11:24,544 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xca78b386d00ea7ed with lease ID 0x56c7673d4851ad62: Processing first storage report for DS-462c86c2-de51-4b7a-801c-2ba3517c9594 from datanode DatanodeRegistration(127.0.0.1:35039, datanodeUuid=723faa33-b9ca-4c30-bbb3-1907a5afaa48, infoPort=34651, infoSecurePort=0, ipcPort=40991, storageInfo=lv=-57;cid=testClusterID;nsid=1181013597;c=1731708682292) 2024-11-15T22:11:24,544 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xca78b386d00ea7ed with lease ID 0x56c7673d4851ad62: from storage DS-462c86c2-de51-4b7a-801c-2ba3517c9594 node DatanodeRegistration(127.0.0.1:35039, datanodeUuid=723faa33-b9ca-4c30-bbb3-1907a5afaa48, infoPort=34651, infoSecurePort=0, ipcPort=40991, storageInfo=lv=-57;cid=testClusterID;nsid=1181013597;c=1731708682292), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T22:11:24,578 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf 2024-11-15T22:11:24,661 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/cluster_048f81ed-d9d5-ea35-fb33-ebaa9399bbca/zookeeper_0, clientPort=56467, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/cluster_048f81ed-d9d5-ea35-fb33-ebaa9399bbca/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/cluster_048f81ed-d9d5-ea35-fb33-ebaa9399bbca/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-15T22:11:24,673 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56467 2024-11-15T22:11:24,684 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:11:24,687 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:11:24,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741825_1001 (size=7) 2024-11-15T22:11:25,401 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31 with version=8 2024-11-15T22:11:25,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/hbase-staging 2024-11-15T22:11:25,544 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-15T22:11:25,840 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ae3311ea5ccb:0 server-side Connection retries=45 2024-11-15T22:11:25,851 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T22:11:25,852 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T22:11:25,857 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T22:11:25,857 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T22:11:25,858 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T22:11:26,024 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-15T22:11:26,100 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-15T22:11:26,113 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-15T22:11:26,118 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T22:11:26,148 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 26297 (auto-detected) 2024-11-15T22:11:26,150 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-15T22:11:26,176 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37303 2024-11-15T22:11:26,207 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37303 connecting to ZooKeeper ensemble=127.0.0.1:56467 2024-11-15T22:11:26,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:373030x0, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T22:11:26,254 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37303-0x100f8747e0b0000 connected 2024-11-15T22:11:26,309 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:11:26,312 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:11:26,330 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T22:11:26,335 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31, hbase.cluster.distributed=false 2024-11-15T22:11:26,384 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T22:11:26,397 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37303 2024-11-15T22:11:26,401 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37303 2024-11-15T22:11:26,402 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37303 2024-11-15T22:11:26,405 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37303 2024-11-15T22:11:26,406 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37303 2024-11-15T22:11:26,561 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ae3311ea5ccb:0 server-side Connection retries=45 2024-11-15T22:11:26,564 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T22:11:26,564 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T22:11:26,564 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T22:11:26,565 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T22:11:26,565 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T22:11:26,569 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T22:11:26,572 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T22:11:26,573 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37917 2024-11-15T22:11:26,575 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37917 connecting to ZooKeeper ensemble=127.0.0.1:56467 2024-11-15T22:11:26,578 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:11:26,582 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:11:26,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:379170x0, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T22:11:26,602 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:379170x0, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T22:11:26,603 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37917-0x100f8747e0b0001 connected 2024-11-15T22:11:26,607 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T22:11:26,620 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T22:11:26,623 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37917-0x100f8747e0b0001, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T22:11:26,630 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37917-0x100f8747e0b0001, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T22:11:26,641 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37917 2024-11-15T22:11:26,644 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37917 2024-11-15T22:11:26,645 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37917 2024-11-15T22:11:26,649 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37917 2024-11-15T22:11:26,650 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37917 2024-11-15T22:11:26,668 DEBUG [M:0;ae3311ea5ccb:37303 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ae3311ea5ccb:37303 2024-11-15T22:11:26,669 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ae3311ea5ccb,37303,1731708685620 2024-11-15T22:11:26,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T22:11:26,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37917-0x100f8747e0b0001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T22:11:26,679 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ae3311ea5ccb,37303,1731708685620 2024-11-15T22:11:26,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37917-0x100f8747e0b0001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T22:11:26,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:11:26,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37917-0x100f8747e0b0001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:11:26,703 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T22:11:26,704 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ae3311ea5ccb,37303,1731708685620 from backup master directory 2024-11-15T22:11:26,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ae3311ea5ccb,37303,1731708685620 2024-11-15T22:11:26,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37917-0x100f8747e0b0001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T22:11:26,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T22:11:26,708 WARN [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T22:11:26,708 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ae3311ea5ccb,37303,1731708685620 2024-11-15T22:11:26,711 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-15T22:11:26,712 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-15T22:11:26,781 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/hbase.id] with ID: cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f 2024-11-15T22:11:26,781 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/.tmp/hbase.id 2024-11-15T22:11:26,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741826_1002 (size=42) 2024-11-15T22:11:26,793 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/.tmp/hbase.id]:[hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/hbase.id] 2024-11-15T22:11:26,851 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:11:26,858 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-15T22:11:26,880 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-15T22:11:26,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37917-0x100f8747e0b0001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:11:26,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:11:26,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741827_1003 (size=196) 2024-11-15T22:11:26,920 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:11:26,923 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-15T22:11:26,941 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:26,945 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T22:11:26,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741828_1004 (size=1189) 2024-11-15T22:11:27,406 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/data/master/store 2024-11-15T22:11:27,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741829_1005 (size=34) 2024-11-15T22:11:27,835 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-15T22:11:27,839 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:11:27,840 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T22:11:27,840 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:11:27,841 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:11:27,843 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T22:11:27,843 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:11:27,843 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:11:27,845 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731708687840Disabling compacts and flushes for region at 1731708687840Disabling writes for close at 1731708687843 (+3 ms)Writing region close event to WAL at 1731708687843Closed at 1731708687843 2024-11-15T22:11:27,848 WARN [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/data/master/store/.initializing 2024-11-15T22:11:27,848 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/WALs/ae3311ea5ccb,37303,1731708685620 2024-11-15T22:11:27,860 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T22:11:27,878 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ae3311ea5ccb%2C37303%2C1731708685620, suffix=, logDir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/WALs/ae3311ea5ccb,37303,1731708685620, archiveDir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/oldWALs, maxLogs=10 2024-11-15T22:11:27,903 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/WALs/ae3311ea5ccb,37303,1731708685620/ae3311ea5ccb%2C37303%2C1731708685620.1731708687883, exclude list is [], retry=0 2024-11-15T22:11:27,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35039,DS-c935f7a9-6b24-44fb-a793-fac06a5ecde6,DISK] 2024-11-15T22:11:27,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-15T22:11:27,967 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/WALs/ae3311ea5ccb,37303,1731708685620/ae3311ea5ccb%2C37303%2C1731708685620.1731708687883 2024-11-15T22:11:27,969 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34651:34651)] 2024-11-15T22:11:27,969 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:11:27,970 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:11:27,974 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:11:27,975 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:11:28,019 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:11:28,048 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-15T22:11:28,053 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:11:28,056 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:11:28,057 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:11:28,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-15T22:11:28,061 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:11:28,062 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:11:28,063 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:11:28,066 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-15T22:11:28,066 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:11:28,067 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:11:28,068 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:11:28,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-15T22:11:28,071 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:11:28,072 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:11:28,073 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:11:28,077 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:11:28,078 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:11:28,084 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:11:28,085 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:11:28,090 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T22:11:28,094 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:11:28,099 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:11:28,100 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65707792, jitterRate=-0.020877599716186523}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T22:11:28,107 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731708687989Initializing all the Stores at 1731708687992 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708687992Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708687993 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708687993Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708687993Cleaning up temporary data from old regions at 1731708688086 (+93 ms)Region opened successfully at 1731708688107 (+21 ms) 2024-11-15T22:11:28,109 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-15T22:11:28,148 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@624917dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ae3311ea5ccb/172.17.0.3:0 2024-11-15T22:11:28,185 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-15T22:11:28,200 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-15T22:11:28,200 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-15T22:11:28,203 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:11:28,205 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T22:11:28,210 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-15T22:11:28,210 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-15T22:11:28,238 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-15T22:11:28,250 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-15T22:11:28,252 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-15T22:11:28,255 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-15T22:11:28,257 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-15T22:11:28,259 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-15T22:11:28,262 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-15T22:11:28,265 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-15T22:11:28,267 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-15T22:11:28,268 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-15T22:11:28,270 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-15T22:11:28,289 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-15T22:11:28,291 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-15T22:11:28,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T22:11:28,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37917-0x100f8747e0b0001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T22:11:28,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:11:28,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37917-0x100f8747e0b0001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:11:28,299 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ae3311ea5ccb,37303,1731708685620, sessionid=0x100f8747e0b0000, setting cluster-up flag (Was=false) 2024-11-15T22:11:28,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:11:28,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37917-0x100f8747e0b0001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:11:28,317 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-15T22:11:28,319 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ae3311ea5ccb,37303,1731708685620 2024-11-15T22:11:28,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:11:28,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37917-0x100f8747e0b0001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:11:28,331 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-15T22:11:28,332 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ae3311ea5ccb,37303,1731708685620 2024-11-15T22:11:28,339 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-15T22:11:28,355 INFO [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(746): ClusterId : cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f 2024-11-15T22:11:28,358 DEBUG [RS:0;ae3311ea5ccb:37917 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T22:11:28,363 DEBUG [RS:0;ae3311ea5ccb:37917 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T22:11:28,364 DEBUG [RS:0;ae3311ea5ccb:37917 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T22:11:28,367 DEBUG [RS:0;ae3311ea5ccb:37917 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T22:11:28,368 DEBUG [RS:0;ae3311ea5ccb:37917 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c137354, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ae3311ea5ccb/172.17.0.3:0 2024-11-15T22:11:28,384 DEBUG [RS:0;ae3311ea5ccb:37917 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ae3311ea5ccb:37917 2024-11-15T22:11:28,388 INFO [RS:0;ae3311ea5ccb:37917 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T22:11:28,388 INFO [RS:0;ae3311ea5ccb:37917 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T22:11:28,388 DEBUG [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T22:11:28,392 INFO [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(2659): reportForDuty to master=ae3311ea5ccb,37303,1731708685620 with port=37917, startcode=1731708686516 2024-11-15T22:11:28,405 DEBUG [RS:0;ae3311ea5ccb:37917 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T22:11:28,424 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:11:28,435 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-15T22:11:28,443 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49303, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T22:11:28,443 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-15T22:11:28,449 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37303 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:28,449 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ae3311ea5ccb,37303,1731708685620 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-15T22:11:28,457 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ae3311ea5ccb:0, corePoolSize=5, maxPoolSize=5 2024-11-15T22:11:28,458 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ae3311ea5ccb:0, corePoolSize=5, maxPoolSize=5 2024-11-15T22:11:28,458 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ae3311ea5ccb:0, corePoolSize=5, maxPoolSize=5 2024-11-15T22:11:28,458 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ae3311ea5ccb:0, corePoolSize=5, maxPoolSize=5 2024-11-15T22:11:28,458 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ae3311ea5ccb:0, corePoolSize=10, maxPoolSize=10 2024-11-15T22:11:28,459 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:11:28,459 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ae3311ea5ccb:0, corePoolSize=2, maxPoolSize=2 2024-11-15T22:11:28,459 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:11:28,465 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708718465 2024-11-15T22:11:28,466 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T22:11:28,466 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-15T22:11:28,467 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-15T22:11:28,468 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-15T22:11:28,473 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-15T22:11:28,473 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-15T22:11:28,474 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-15T22:11:28,474 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-15T22:11:28,474 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:11:28,474 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-15T22:11:28,479 DEBUG [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-15T22:11:28,479 WARN [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-15T22:11:28,478 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T22:11:28,482 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-15T22:11:28,483 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-15T22:11:28,484 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-15T22:11:28,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741831_1007 (size=1321) 2024-11-15T22:11:28,486 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-15T22:11:28,486 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-15T22:11:28,489 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.large.0-1731708688488,5,FailOnTimeoutGroup] 2024-11-15T22:11:28,489 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.small.0-1731708688489,5,FailOnTimeoutGroup] 2024-11-15T22:11:28,490 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T22:11:28,490 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-15T22:11:28,491 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-15T22:11:28,491 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-15T22:11:28,581 INFO [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(2659): reportForDuty to master=ae3311ea5ccb,37303,1731708685620 with port=37917, startcode=1731708686516 2024-11-15T22:11:28,584 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37303 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:28,587 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37303 {}] master.ServerManager(517): Registering regionserver=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:28,595 DEBUG [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31 2024-11-15T22:11:28,595 DEBUG [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35615 2024-11-15T22:11:28,596 DEBUG [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T22:11:28,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T22:11:28,600 DEBUG [RS:0;ae3311ea5ccb:37917 {}] zookeeper.ZKUtil(111): regionserver:37917-0x100f8747e0b0001, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:28,601 WARN [RS:0;ae3311ea5ccb:37917 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T22:11:28,601 INFO [RS:0;ae3311ea5ccb:37917 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T22:11:28,601 DEBUG [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/WALs/ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:28,603 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ae3311ea5ccb,37917,1731708686516] 2024-11-15T22:11:28,629 INFO [RS:0;ae3311ea5ccb:37917 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T22:11:28,644 INFO [RS:0;ae3311ea5ccb:37917 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T22:11:28,649 INFO [RS:0;ae3311ea5ccb:37917 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T22:11:28,649 INFO [RS:0;ae3311ea5ccb:37917 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T22:11:28,650 INFO [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T22:11:28,656 INFO [RS:0;ae3311ea5ccb:37917 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T22:11:28,657 INFO [RS:0;ae3311ea5ccb:37917 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T22:11:28,657 DEBUG [RS:0;ae3311ea5ccb:37917 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:11:28,658 DEBUG [RS:0;ae3311ea5ccb:37917 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:11:28,658 DEBUG [RS:0;ae3311ea5ccb:37917 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:11:28,658 DEBUG [RS:0;ae3311ea5ccb:37917 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:11:28,658 DEBUG [RS:0;ae3311ea5ccb:37917 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:11:28,658 DEBUG [RS:0;ae3311ea5ccb:37917 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0, corePoolSize=2, maxPoolSize=2 2024-11-15T22:11:28,659 DEBUG [RS:0;ae3311ea5ccb:37917 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:11:28,659 DEBUG [RS:0;ae3311ea5ccb:37917 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:11:28,659 DEBUG [RS:0;ae3311ea5ccb:37917 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:11:28,659 DEBUG [RS:0;ae3311ea5ccb:37917 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:11:28,659 DEBUG [RS:0;ae3311ea5ccb:37917 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:11:28,660 DEBUG [RS:0;ae3311ea5ccb:37917 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:11:28,660 DEBUG [RS:0;ae3311ea5ccb:37917 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ae3311ea5ccb:0, corePoolSize=3, maxPoolSize=3 2024-11-15T22:11:28,660 DEBUG [RS:0;ae3311ea5ccb:37917 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0, corePoolSize=3, maxPoolSize=3 2024-11-15T22:11:28,661 INFO [RS:0;ae3311ea5ccb:37917 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T22:11:28,661 INFO [RS:0;ae3311ea5ccb:37917 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T22:11:28,661 INFO [RS:0;ae3311ea5ccb:37917 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T22:11:28,661 INFO [RS:0;ae3311ea5ccb:37917 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T22:11:28,661 INFO [RS:0;ae3311ea5ccb:37917 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T22:11:28,662 INFO [RS:0;ae3311ea5ccb:37917 {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,37917,1731708686516-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T22:11:28,682 INFO [RS:0;ae3311ea5ccb:37917 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T22:11:28,684 INFO [RS:0;ae3311ea5ccb:37917 {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,37917,1731708686516-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T22:11:28,684 INFO [RS:0;ae3311ea5ccb:37917 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:11:28,685 INFO [RS:0;ae3311ea5ccb:37917 {}] regionserver.Replication(171): ae3311ea5ccb,37917,1731708686516 started 2024-11-15T22:11:28,705 INFO [RS:0;ae3311ea5ccb:37917 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:11:28,705 INFO [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(1482): Serving as ae3311ea5ccb,37917,1731708686516, RpcServer on ae3311ea5ccb/172.17.0.3:37917, sessionid=0x100f8747e0b0001 2024-11-15T22:11:28,706 DEBUG [RS:0;ae3311ea5ccb:37917 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T22:11:28,706 DEBUG [RS:0;ae3311ea5ccb:37917 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:28,707 DEBUG [RS:0;ae3311ea5ccb:37917 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ae3311ea5ccb,37917,1731708686516' 2024-11-15T22:11:28,707 DEBUG [RS:0;ae3311ea5ccb:37917 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T22:11:28,708 DEBUG [RS:0;ae3311ea5ccb:37917 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T22:11:28,709 DEBUG [RS:0;ae3311ea5ccb:37917 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T22:11:28,709 DEBUG [RS:0;ae3311ea5ccb:37917 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T22:11:28,709 DEBUG [RS:0;ae3311ea5ccb:37917 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:28,709 DEBUG [RS:0;ae3311ea5ccb:37917 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ae3311ea5ccb,37917,1731708686516' 2024-11-15T22:11:28,709 DEBUG [RS:0;ae3311ea5ccb:37917 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T22:11:28,710 DEBUG [RS:0;ae3311ea5ccb:37917 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T22:11:28,710 DEBUG [RS:0;ae3311ea5ccb:37917 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T22:11:28,711 INFO [RS:0;ae3311ea5ccb:37917 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T22:11:28,711 INFO [RS:0;ae3311ea5ccb:37917 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T22:11:28,817 INFO [RS:0;ae3311ea5ccb:37917 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T22:11:28,821 INFO [RS:0;ae3311ea5ccb:37917 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ae3311ea5ccb%2C37917%2C1731708686516, suffix=, logDir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/WALs/ae3311ea5ccb,37917,1731708686516, archiveDir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/oldWALs, maxLogs=32 2024-11-15T22:11:28,838 DEBUG [RS:0;ae3311ea5ccb:37917 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/WALs/ae3311ea5ccb,37917,1731708686516/ae3311ea5ccb%2C37917%2C1731708686516.1731708688823, exclude list is [], retry=0 2024-11-15T22:11:28,843 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35039,DS-c935f7a9-6b24-44fb-a793-fac06a5ecde6,DISK] 2024-11-15T22:11:28,846 INFO [RS:0;ae3311ea5ccb:37917 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/WALs/ae3311ea5ccb,37917,1731708686516/ae3311ea5ccb%2C37917%2C1731708686516.1731708688823 2024-11-15T22:11:28,847 DEBUG [RS:0;ae3311ea5ccb:37917 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34651:34651)] 2024-11-15T22:11:28,887 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-15T22:11:28,887 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31 2024-11-15T22:11:28,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741833_1009 (size=32) 2024-11-15T22:11:28,898 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:11:28,901 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T22:11:28,903 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T22:11:28,903 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:11:28,904 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:11:28,904 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T22:11:28,907 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T22:11:28,907 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:11:28,908 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:11:28,908 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T22:11:28,910 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T22:11:28,910 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:11:28,911 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:11:28,911 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T22:11:28,914 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T22:11:28,914 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:11:28,916 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:11:28,916 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T22:11:28,918 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740 2024-11-15T22:11:28,919 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740 2024-11-15T22:11:28,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T22:11:28,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T22:11:28,924 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:11:28,927 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T22:11:28,932 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:11:28,933 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74131197, jitterRate=0.1046409159898758}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:11:28,936 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731708688899Initializing all the Stores at 1731708688900 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708688900Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708688900Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708688900Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708688900Cleaning up temporary data from old regions at 1731708688923 (+23 ms)Region opened successfully at 1731708688936 (+13 ms) 2024-11-15T22:11:28,936 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T22:11:28,936 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T22:11:28,936 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T22:11:28,937 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T22:11:28,937 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T22:11:28,939 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T22:11:28,939 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731708688936Disabling compacts and flushes for region at 1731708688936Disabling writes for close at 1731708688937 (+1 ms)Writing region close event to WAL at 1731708688938 (+1 ms)Closed at 1731708688939 (+1 ms) 2024-11-15T22:11:28,943 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T22:11:28,944 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-15T22:11:28,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T22:11:28,960 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:11:28,963 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-15T22:11:29,116 DEBUG [ae3311ea5ccb:37303 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-15T22:11:29,130 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:29,139 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ae3311ea5ccb,37917,1731708686516, state=OPENING 2024-11-15T22:11:29,145 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-15T22:11:29,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37917-0x100f8747e0b0001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:11:29,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:11:29,148 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:11:29,148 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:11:29,150 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:11:29,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,37917,1731708686516}] 2024-11-15T22:11:29,332 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T22:11:29,338 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34691, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T22:11:29,354 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T22:11:29,354 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T22:11:29,355 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T22:11:29,359 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ae3311ea5ccb%2C37917%2C1731708686516.meta, suffix=.meta, logDir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/WALs/ae3311ea5ccb,37917,1731708686516, archiveDir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/oldWALs, maxLogs=32 2024-11-15T22:11:29,378 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/WALs/ae3311ea5ccb,37917,1731708686516/ae3311ea5ccb%2C37917%2C1731708686516.meta.1731708689361.meta, exclude list is [], retry=0 2024-11-15T22:11:29,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35039,DS-c935f7a9-6b24-44fb-a793-fac06a5ecde6,DISK] 2024-11-15T22:11:29,385 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/WALs/ae3311ea5ccb,37917,1731708686516/ae3311ea5ccb%2C37917%2C1731708686516.meta.1731708689361.meta 2024-11-15T22:11:29,386 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34651:34651)] 2024-11-15T22:11:29,386 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:11:29,388 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T22:11:29,391 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T22:11:29,396 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T22:11:29,402 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T22:11:29,402 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:11:29,402 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T22:11:29,403 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T22:11:29,406 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T22:11:29,407 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T22:11:29,408 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:11:29,408 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:11:29,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T22:11:29,410 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T22:11:29,410 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:11:29,411 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:11:29,411 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T22:11:29,412 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T22:11:29,413 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:11:29,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:11:29,414 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T22:11:29,415 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T22:11:29,415 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:11:29,416 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:11:29,416 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T22:11:29,418 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740 2024-11-15T22:11:29,421 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740 2024-11-15T22:11:29,423 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T22:11:29,423 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T22:11:29,424 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:11:29,427 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T22:11:29,428 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71581836, jitterRate=0.06665247678756714}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:11:29,428 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T22:11:29,430 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731708689403Writing region info on filesystem at 1731708689403Initializing all the Stores at 1731708689405 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708689405Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708689405Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708689405Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708689406 (+1 ms)Cleaning up temporary data from old regions at 1731708689423 (+17 ms)Running coprocessor post-open hooks at 1731708689428 (+5 ms)Region opened successfully at 1731708689430 (+2 ms) 2024-11-15T22:11:29,438 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731708689321 2024-11-15T22:11:29,452 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T22:11:29,452 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T22:11:29,454 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:29,456 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ae3311ea5ccb,37917,1731708686516, state=OPEN 2024-11-15T22:11:29,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37917-0x100f8747e0b0001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:11:29,473 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:11:29,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:11:29,474 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:11:29,476 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:29,499 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-15T22:11:29,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,37917,1731708686516 in 324 msec 2024-11-15T22:11:29,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-15T22:11:29,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 550 msec 2024-11-15T22:11:29,512 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T22:11:29,512 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-15T22:11:29,538 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:11:29,539 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:11:29,564 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:11:29,568 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58543, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:11:29,594 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2210 sec 2024-11-15T22:11:29,594 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731708689594, completionTime=-1 2024-11-15T22:11:29,598 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-15T22:11:29,598 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T22:11:29,634 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T22:11:29,634 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708749634 2024-11-15T22:11:29,634 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731708809634 2024-11-15T22:11:29,635 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 36 msec 2024-11-15T22:11:29,638 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,37303,1731708685620-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T22:11:29,638 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,37303,1731708685620-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:11:29,638 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,37303,1731708685620-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:11:29,671 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ae3311ea5ccb:37303, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:11:29,674 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-15T22:11:29,677 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-15T22:11:29,683 DEBUG [master/ae3311ea5ccb:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-15T22:11:29,730 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.022sec 2024-11-15T22:11:29,732 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-15T22:11:29,733 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-15T22:11:29,734 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-15T22:11:29,735 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-15T22:11:29,735 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-15T22:11:29,736 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,37303,1731708685620-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T22:11:29,736 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,37303,1731708685620-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-15T22:11:29,750 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-15T22:11:29,751 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-15T22:11:29,752 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,37303,1731708685620-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:11:29,787 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7322f5f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:29,796 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-15T22:11:29,796 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-15T22:11:29,806 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:11:29,814 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:11:29,826 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:11:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:11:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:11:29,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5803a924, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:29,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:11:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:11:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:29,839 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:11:29,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f328d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:29,842 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:11:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:11:29,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:11:29,866 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:11:29,899 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ae3311ea5ccb,37303,1731708685620 2024-11-15T22:11:29,917 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=535, ProcessCount=11, AvailableMemoryMB=4099 2024-11-15T22:11:29,926 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T22:11:29,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ae3311ea5ccb,37303,1731708685620 2024-11-15T22:11:29,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5164c17f 2024-11-15T22:11:29,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T22:11:29,940 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T22:11:29,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T22:11:29,948 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:11:29,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T22:11:29,961 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T22:11:29,964 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-15T22:11:29,964 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:11:29,967 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T22:11:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T22:11:29,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741835_1011 (size=959) 2024-11-15T22:11:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T22:11:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T22:11:30,387 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31 2024-11-15T22:11:30,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741836_1012 (size=53) 2024-11-15T22:11:30,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T22:11:30,800 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:11:30,800 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3220684e1e7cfe8432a04d7ee5eae414, disabling compactions & flushes 2024-11-15T22:11:30,800 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:30,800 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:30,800 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. after waiting 0 ms 2024-11-15T22:11:30,800 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:30,800 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:30,800 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3220684e1e7cfe8432a04d7ee5eae414: Waiting for close lock at 1731708690800Disabling compacts and flushes for region at 1731708690800Disabling writes for close at 1731708690800Writing region close event to WAL at 1731708690800Closed at 1731708690800 2024-11-15T22:11:30,803 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T22:11:30,809 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731708690803"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708690803"}]},"ts":"1731708690803"} 2024-11-15T22:11:30,815 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T22:11:30,817 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T22:11:30,820 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708690817"}]},"ts":"1731708690817"} 2024-11-15T22:11:30,824 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T22:11:30,826 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3220684e1e7cfe8432a04d7ee5eae414, ASSIGN}] 2024-11-15T22:11:30,828 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3220684e1e7cfe8432a04d7ee5eae414, ASSIGN 2024-11-15T22:11:30,830 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3220684e1e7cfe8432a04d7ee5eae414, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,37917,1731708686516; forceNewPlan=false, retain=false 2024-11-15T22:11:30,982 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3220684e1e7cfe8432a04d7ee5eae414, regionState=OPENING, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:30,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3220684e1e7cfe8432a04d7ee5eae414, ASSIGN because future has completed 2024-11-15T22:11:30,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516}] 2024-11-15T22:11:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T22:11:31,155 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:31,155 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:11:31,156 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:31,156 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:11:31,156 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:31,156 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:31,165 INFO [StoreOpener-3220684e1e7cfe8432a04d7ee5eae414-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:31,174 INFO [StoreOpener-3220684e1e7cfe8432a04d7ee5eae414-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3220684e1e7cfe8432a04d7ee5eae414 columnFamilyName A 2024-11-15T22:11:31,174 DEBUG [StoreOpener-3220684e1e7cfe8432a04d7ee5eae414-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:11:31,176 INFO [StoreOpener-3220684e1e7cfe8432a04d7ee5eae414-1 {}] regionserver.HStore(327): Store=3220684e1e7cfe8432a04d7ee5eae414/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:11:31,176 INFO [StoreOpener-3220684e1e7cfe8432a04d7ee5eae414-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:31,179 INFO [StoreOpener-3220684e1e7cfe8432a04d7ee5eae414-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3220684e1e7cfe8432a04d7ee5eae414 columnFamilyName B 2024-11-15T22:11:31,179 DEBUG [StoreOpener-3220684e1e7cfe8432a04d7ee5eae414-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:11:31,180 INFO [StoreOpener-3220684e1e7cfe8432a04d7ee5eae414-1 {}] regionserver.HStore(327): Store=3220684e1e7cfe8432a04d7ee5eae414/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:11:31,180 INFO [StoreOpener-3220684e1e7cfe8432a04d7ee5eae414-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:31,183 INFO [StoreOpener-3220684e1e7cfe8432a04d7ee5eae414-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3220684e1e7cfe8432a04d7ee5eae414 columnFamilyName C 2024-11-15T22:11:31,183 DEBUG [StoreOpener-3220684e1e7cfe8432a04d7ee5eae414-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:11:31,184 INFO [StoreOpener-3220684e1e7cfe8432a04d7ee5eae414-1 {}] regionserver.HStore(327): Store=3220684e1e7cfe8432a04d7ee5eae414/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:11:31,185 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:31,186 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:31,187 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:31,189 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:31,189 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:31,191 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:11:31,193 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:31,205 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:11:31,207 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 3220684e1e7cfe8432a04d7ee5eae414; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75054965, jitterRate=0.11840613186359406}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:11:31,207 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:31,208 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 3220684e1e7cfe8432a04d7ee5eae414: Running coprocessor pre-open hook at 1731708691157Writing region info on filesystem at 1731708691157Initializing all the Stores at 1731708691160 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708691160Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708691165 (+5 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708691165Cleaning up temporary data from old regions at 1731708691190 (+25 ms)Running coprocessor post-open hooks at 1731708691207 (+17 ms)Region opened successfully at 1731708691208 (+1 ms) 2024-11-15T22:11:31,210 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., pid=6, masterSystemTime=1731708691144 2024-11-15T22:11:31,214 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:31,214 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:31,215 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3220684e1e7cfe8432a04d7ee5eae414, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:31,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 because future has completed 2024-11-15T22:11:31,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-15T22:11:31,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 in 238 msec 2024-11-15T22:11:31,238 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-15T22:11:31,238 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3220684e1e7cfe8432a04d7ee5eae414, ASSIGN in 406 msec 2024-11-15T22:11:31,239 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T22:11:31,240 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708691240"}]},"ts":"1731708691240"} 2024-11-15T22:11:31,243 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T22:11:31,245 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T22:11:31,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2940 sec 2024-11-15T22:11:32,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T22:11:32,129 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:11:32,135 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9032552, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,135 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:11:32,135 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:11:32,137 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:11:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:11:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:11:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d6336a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:11:32,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:11:32,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:32,144 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:11:32,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d7323ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,146 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:11:32,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:11:32,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:11:32,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:11:32,153 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T22:11:32,155 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is ae3311ea5ccb,37303,1731708685620 2024-11-15T22:11:32,155 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@58b5dcfd 2024-11-15T22:11:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T22:11:32,161 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T22:11:32,167 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@776dd602, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,167 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:11:32,167 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:11:32,169 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:11:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:11:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:11:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f338c72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:11:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:11:32,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:32,174 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56042, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:11:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4184659e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a0238d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,177 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:11:32,177 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:11:32,179 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:11:32,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:11:32,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:11:32,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ffd0298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:11:32,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:11:32,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:32,183 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:11:32,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30bed14c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@241fa23d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,188 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:11:32,188 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:11:32,190 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:11:32,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:11:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:11:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af2c0a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:11:32,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:11:32,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:32,195 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:11:32,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1612488f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c5ab0ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,202 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:11:32,202 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:11:32,215 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:11:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:11:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:11:32,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b9dd8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:11:32,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:11:32,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:32,219 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:11:32,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3ac293, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@566e69dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:11:32,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:11:32,222 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:11:32,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:11:32,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:11:32,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f99b884, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:11:32,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:11:32,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:32,225 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:11:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75abeb9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d577fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,227 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:11:32,228 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:11:32,229 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:11:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:11:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:11:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@598c3d89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:11:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:11:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:32,232 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:11:32,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@154b423c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d715406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,238 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:11:32,238 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:11:32,239 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:11:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:11:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:11:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24aa1c15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:11:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:11:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:32,242 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:11:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fce0532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59025020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,246 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:11:32,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:11:32,247 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:11:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:11:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:11:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ee22a4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:11:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:11:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:32,249 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:11:32,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23451635, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b966c63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,253 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:11:32,254 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:11:32,255 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:11:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:11:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:11:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62078f75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:11:32,256 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:11:32,256 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:32,257 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:11:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145f578c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:11:32,278 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:11:32,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:11:32,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:11:32,282 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:11:32,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:11:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:11:32,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:11:32,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:11:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:11:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:11:32,294 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:11:32,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:11:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:11:32,298 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:11:32,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:11:32,319 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:11:32,320 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:11:32,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:11:32,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:11:32,323 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:11:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:11:32,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:11:32,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:11:32,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:11:32,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:11:32,339 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:11:32,340 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:11:32,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:11:32,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:11:32,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:11:32,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:11:32,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:11:32,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:11:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:11:32,349 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:11:32,354 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:11:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:11:32,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:11:32,358 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:11:32,360 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:11:32,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:11:32,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:11:32,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:11:32,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:11:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:11:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-15T22:11:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:11:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T22:11:32,374 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:11:32,381 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:11:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:32,384 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:11:32,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3220684e1e7cfe8432a04d7ee5eae414 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T22:11:32,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:11:32,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T22:11:32,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/bd8754ef173e40229c2fe87f0349e03e is 50, key is test_row_0/A:col10/1731708692378/Put/seqid=0 2024-11-15T22:11:32,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:11:32,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:32,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:32,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:32,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:32,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741837_1013 (size=14341) 2024-11-15T22:11:32,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:32,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/bd8754ef173e40229c2fe87f0349e03e 2024-11-15T22:11:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:32,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:32,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708752611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:32,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:32,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:32,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:32,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708752617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:32,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708752625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:32,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:32,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:32,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:32,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:32,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:32,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:32,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708752631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:32,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:32,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:32,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:32,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708752636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:32,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:32,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:32,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:32,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T22:11:32,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/b09c4538f2bc43f3b01fb985716f333d is 50, key is test_row_0/B:col10/1731708692378/Put/seqid=0 2024-11-15T22:11:32,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741838_1014 (size=12001) 2024-11-15T22:11:32,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/b09c4538f2bc43f3b01fb985716f333d 2024-11-15T22:11:32,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:32,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708752757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:32,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:32,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:32,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:32,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:32,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708752758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:32,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:11:32,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:32,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708752759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:32,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:32,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:32,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:32,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:32,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:32,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:32,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708752759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:32,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:32,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:32,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:32,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:32,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:32,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:32,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:32,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:32,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708752767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:32,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:32,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:32,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:32,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/baf192ea83744b11a5d43814b6b819de is 50, key is test_row_0/C:col10/1731708692378/Put/seqid=0 2024-11-15T22:11:32,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741839_1015 (size=12001) 2024-11-15T22:11:32,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/baf192ea83744b11a5d43814b6b819de 2024-11-15T22:11:32,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/bd8754ef173e40229c2fe87f0349e03e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/bd8754ef173e40229c2fe87f0349e03e 2024-11-15T22:11:32,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/bd8754ef173e40229c2fe87f0349e03e, entries=200, sequenceid=13, filesize=14.0 K 2024-11-15T22:11:32,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/b09c4538f2bc43f3b01fb985716f333d as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/b09c4538f2bc43f3b01fb985716f333d 2024-11-15T22:11:32,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/b09c4538f2bc43f3b01fb985716f333d, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T22:11:32,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/baf192ea83744b11a5d43814b6b819de as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/baf192ea83744b11a5d43814b6b819de 2024-11-15T22:11:32,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/baf192ea83744b11a5d43814b6b819de, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T22:11:32,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:11:32,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:32,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:32,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:32,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:32,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:32,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3220684e1e7cfe8432a04d7ee5eae414 in 551ms, sequenceid=13, compaction requested=false 2024-11-15T22:11:32,949 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T22:11:32,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:32,997 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:11:32,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3220684e1e7cfe8432a04d7ee5eae414 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T22:11:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T22:11:33,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/741434a0e7f44bcb826fffad8632a7e0 is 50, key is test_row_0/A:col10/1731708692607/Put/seqid=0 2024-11-15T22:11:33,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708753029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708753037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708753042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708753040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708753046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741840_1016 (size=12001) 2024-11-15T22:11:33,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/741434a0e7f44bcb826fffad8632a7e0 2024-11-15T22:11:33,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:11:33,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:33,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:33,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:33,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:33,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:33,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/f5d4d428df354d99858aad10492887b0 is 50, key is test_row_0/B:col10/1731708692607/Put/seqid=0 2024-11-15T22:11:33,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:33,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741841_1017 (size=12001) 2024-11-15T22:11:33,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/f5d4d428df354d99858aad10492887b0 2024-11-15T22:11:33,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708753157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/f8730071c28544f4bf6c5db4264f6e97 is 50, key is test_row_0/C:col10/1731708692607/Put/seqid=0 2024-11-15T22:11:33,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708753159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708753168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708753169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708753167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741842_1018 (size=12001) 2024-11-15T22:11:33,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/f8730071c28544f4bf6c5db4264f6e97 2024-11-15T22:11:33,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:11:33,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:33,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:33,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:33,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:33,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:33,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:33,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/741434a0e7f44bcb826fffad8632a7e0 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/741434a0e7f44bcb826fffad8632a7e0 2024-11-15T22:11:33,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/741434a0e7f44bcb826fffad8632a7e0, entries=150, sequenceid=39, filesize=11.7 K 2024-11-15T22:11:33,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/f5d4d428df354d99858aad10492887b0 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/f5d4d428df354d99858aad10492887b0 2024-11-15T22:11:33,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/f5d4d428df354d99858aad10492887b0, entries=150, sequenceid=39, filesize=11.7 K 2024-11-15T22:11:33,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/f8730071c28544f4bf6c5db4264f6e97 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/f8730071c28544f4bf6c5db4264f6e97 2024-11-15T22:11:33,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/f8730071c28544f4bf6c5db4264f6e97, entries=150, sequenceid=39, filesize=11.7 K 2024-11-15T22:11:33,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 3220684e1e7cfe8432a04d7ee5eae414 in 360ms, sequenceid=39, compaction requested=false 2024-11-15T22:11:33,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:33,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:33,383 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:11:33,383 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3220684e1e7cfe8432a04d7ee5eae414 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T22:11:33,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/a3006ef71f2145b2bcc639aa836d35f1 is 50, key is test_row_0/A:col10/1731708693379/Put/seqid=0 2024-11-15T22:11:33,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:11:33,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:33,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:33,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:33,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:33,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:33,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:33,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741843_1019 (size=14341) 2024-11-15T22:11:33,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/a3006ef71f2145b2bcc639aa836d35f1 2024-11-15T22:11:33,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/cb0855ad35304ecfb1dee5c5a50b9a23 is 50, key is test_row_0/B:col10/1731708693379/Put/seqid=0 2024-11-15T22:11:33,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T22:11:33,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708753502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741844_1020 (size=12001) 2024-11-15T22:11:33,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/cb0855ad35304ecfb1dee5c5a50b9a23 2024-11-15T22:11:33,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708753508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708753509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708753515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708753516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/49dd085f71854997b3b3d52fc25e6d13 is 50, key is test_row_0/C:col10/1731708693379/Put/seqid=0 2024-11-15T22:11:33,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:11:33,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:33,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:33,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:33,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:33,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:33,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708753638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741845_1021 (size=12001) 2024-11-15T22:11:33,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708753647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708753657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708753658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708753659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:11:33,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:33,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:33,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:33,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:33,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:33,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:33,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708753847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708753867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708753877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708753890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:33,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708753891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:33,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:11:33,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:33,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:33,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:33,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:33,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:33,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:34,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/49dd085f71854997b3b3d52fc25e6d13 2024-11-15T22:11:34,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/a3006ef71f2145b2bcc639aa836d35f1 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/a3006ef71f2145b2bcc639aa836d35f1 2024-11-15T22:11:34,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:11:34,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:34,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:34,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:34,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:34,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:34,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:34,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/a3006ef71f2145b2bcc639aa836d35f1, entries=200, sequenceid=51, filesize=14.0 K 2024-11-15T22:11:34,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/cb0855ad35304ecfb1dee5c5a50b9a23 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/cb0855ad35304ecfb1dee5c5a50b9a23 2024-11-15T22:11:34,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/cb0855ad35304ecfb1dee5c5a50b9a23, entries=150, sequenceid=51, filesize=11.7 K 2024-11-15T22:11:34,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/49dd085f71854997b3b3d52fc25e6d13 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/49dd085f71854997b3b3d52fc25e6d13 2024-11-15T22:11:34,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/49dd085f71854997b3b3d52fc25e6d13, entries=150, sequenceid=51, filesize=11.7 K 2024-11-15T22:11:34,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3220684e1e7cfe8432a04d7ee5eae414 in 775ms, sequenceid=51, compaction requested=true 2024-11-15T22:11:34,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:34,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3220684e1e7cfe8432a04d7ee5eae414:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:11:34,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:34,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:34,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3220684e1e7cfe8432a04d7ee5eae414:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:11:34,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:34,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3220684e1e7cfe8432a04d7ee5eae414:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:11:34,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:11:34,173 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:11:34,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3220684e1e7cfe8432a04d7ee5eae414 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T22:11:34,176 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:11:34,179 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:11:34,181 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:11:34,183 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 3220684e1e7cfe8432a04d7ee5eae414/B is initiating minor compaction (all files) 2024-11-15T22:11:34,184 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3220684e1e7cfe8432a04d7ee5eae414/B in TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:34,184 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/b09c4538f2bc43f3b01fb985716f333d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/f5d4d428df354d99858aad10492887b0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/cb0855ad35304ecfb1dee5c5a50b9a23] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp, totalSize=35.2 K 2024-11-15T22:11:34,187 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting b09c4538f2bc43f3b01fb985716f333d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731708692358 2024-11-15T22:11:34,188 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting f5d4d428df354d99858aad10492887b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731708692607 2024-11-15T22:11:34,189 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting cb0855ad35304ecfb1dee5c5a50b9a23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731708693035 2024-11-15T22:11:34,195 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:11:34,195 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 3220684e1e7cfe8432a04d7ee5eae414/A is initiating minor compaction (all files) 2024-11-15T22:11:34,195 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3220684e1e7cfe8432a04d7ee5eae414/A in TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:34,196 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/bd8754ef173e40229c2fe87f0349e03e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/741434a0e7f44bcb826fffad8632a7e0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/a3006ef71f2145b2bcc639aa836d35f1] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp, totalSize=39.7 K 2024-11-15T22:11:34,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/f199ca5c8e434bc7b5f0261444b3445f is 50, key is test_row_0/A:col10/1731708693490/Put/seqid=0 2024-11-15T22:11:34,203 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd8754ef173e40229c2fe87f0349e03e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731708692355 2024-11-15T22:11:34,208 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 741434a0e7f44bcb826fffad8632a7e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731708692607 2024-11-15T22:11:34,222 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3006ef71f2145b2bcc639aa836d35f1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731708693004 2024-11-15T22:11:34,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741846_1022 (size=14341) 2024-11-15T22:11:34,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:11:34,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:34,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:34,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:34,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:34,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:34,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:34,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:34,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708754239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:34,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:34,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708754246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:34,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:34,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:34,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708754253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:34,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:34,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:34,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708754254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:34,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:34,278 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3220684e1e7cfe8432a04d7ee5eae414#B#compaction#10 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:11:34,279 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/babbd67dceeb4d67b17828eac2ffd4d8 is 50, key is test_row_0/B:col10/1731708693379/Put/seqid=0 2024-11-15T22:11:34,289 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3220684e1e7cfe8432a04d7ee5eae414#A#compaction#11 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:11:34,290 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/043bb49612264f99bc5579e77792909b is 50, key is test_row_0/A:col10/1731708693379/Put/seqid=0 2024-11-15T22:11:34,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:34,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708754260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:34,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:34,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741847_1023 (size=12104) 2024-11-15T22:11:34,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741848_1024 (size=12104) 2024-11-15T22:11:34,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:34,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708754368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:34,374 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/043bb49612264f99bc5579e77792909b as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/043bb49612264f99bc5579e77792909b 2024-11-15T22:11:34,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:34,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708754369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:34,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:34,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708754377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:34,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:34,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708754377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:34,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:34,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:34,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708754397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:34,416 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3220684e1e7cfe8432a04d7ee5eae414/A of 3220684e1e7cfe8432a04d7ee5eae414 into 043bb49612264f99bc5579e77792909b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:11:34,416 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:34,419 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., storeName=3220684e1e7cfe8432a04d7ee5eae414/A, priority=13, startTime=1731708694161; duration=0sec 2024-11-15T22:11:34,420 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:11:34,420 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3220684e1e7cfe8432a04d7ee5eae414:A 2024-11-15T22:11:34,420 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:11:34,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:11:34,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:34,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:34,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:34,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:34,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:34,423 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:11:34,423 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 3220684e1e7cfe8432a04d7ee5eae414/C is initiating minor compaction (all files) 2024-11-15T22:11:34,423 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3220684e1e7cfe8432a04d7ee5eae414/C in TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:34,423 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/baf192ea83744b11a5d43814b6b819de, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/f8730071c28544f4bf6c5db4264f6e97, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/49dd085f71854997b3b3d52fc25e6d13] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp, totalSize=35.2 K 2024-11-15T22:11:34,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:34,425 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting baf192ea83744b11a5d43814b6b819de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731708692358 2024-11-15T22:11:34,427 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8730071c28544f4bf6c5db4264f6e97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731708692607 2024-11-15T22:11:34,428 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49dd085f71854997b3b3d52fc25e6d13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731708693035 2024-11-15T22:11:34,470 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3220684e1e7cfe8432a04d7ee5eae414#C#compaction#12 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:11:34,471 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/45189ca8d83347d398a3c1b9bca862a1 is 50, key is test_row_0/C:col10/1731708693379/Put/seqid=0 2024-11-15T22:11:34,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741849_1025 (size=12104) 2024-11-15T22:11:34,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T22:11:34,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:11:34,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:34,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:34,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:34,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:34,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:34,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:34,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708754581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:34,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708754587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:34,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708754597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:34,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:34,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708754597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:34,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:34,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:34,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708754617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:34,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:34,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/f199ca5c8e434bc7b5f0261444b3445f 2024-11-15T22:11:34,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/b84685ef7f62405f8abf39cde76ec4fd is 50, key is test_row_0/B:col10/1731708693490/Put/seqid=0 2024-11-15T22:11:34,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741850_1026 (size=12001) 2024-11-15T22:11:34,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/b84685ef7f62405f8abf39cde76ec4fd 2024-11-15T22:11:34,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:11:34,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:34,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:34,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:34,760 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/babbd67dceeb4d67b17828eac2ffd4d8 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/babbd67dceeb4d67b17828eac2ffd4d8 2024-11-15T22:11:34,774 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3220684e1e7cfe8432a04d7ee5eae414/B of 3220684e1e7cfe8432a04d7ee5eae414 into babbd67dceeb4d67b17828eac2ffd4d8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:11:34,774 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:34,774 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., storeName=3220684e1e7cfe8432a04d7ee5eae414/B, priority=13, startTime=1731708694172; duration=0sec 2024-11-15T22:11:34,774 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:34,774 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3220684e1e7cfe8432a04d7ee5eae414:B 2024-11-15T22:11:34,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/cb54fdd5050c4042a518e874fe8923d7 is 50, key is test_row_0/C:col10/1731708693490/Put/seqid=0 2024-11-15T22:11:34,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741851_1027 (size=12001) 2024-11-15T22:11:34,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708754897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:34,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:34,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:11:34,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708754907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:34,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708754907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:34,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:34,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:34,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708754908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:34,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:34,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:34,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:34,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:34,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:34,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:34,930 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/45189ca8d83347d398a3c1b9bca862a1 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/45189ca8d83347d398a3c1b9bca862a1 2024-11-15T22:11:34,936 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T22:11:34,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708754927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:34,943 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3220684e1e7cfe8432a04d7ee5eae414/C of 3220684e1e7cfe8432a04d7ee5eae414 into 45189ca8d83347d398a3c1b9bca862a1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:11:34,943 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:34,943 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., storeName=3220684e1e7cfe8432a04d7ee5eae414/C, priority=13, startTime=1731708694172; duration=0sec 2024-11-15T22:11:34,943 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:34,944 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3220684e1e7cfe8432a04d7ee5eae414:C 2024-11-15T22:11:35,055 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-15T22:11:35,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:11:35,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:35,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:35,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:35,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:35,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:35,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:11:35,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:35,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:35,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:35,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:35,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:35,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:35,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/cb54fdd5050c4042a518e874fe8923d7 2024-11-15T22:11:35,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/f199ca5c8e434bc7b5f0261444b3445f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/f199ca5c8e434bc7b5f0261444b3445f 2024-11-15T22:11:35,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/f199ca5c8e434bc7b5f0261444b3445f, entries=200, sequenceid=76, filesize=14.0 K 2024-11-15T22:11:35,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/b84685ef7f62405f8abf39cde76ec4fd as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/b84685ef7f62405f8abf39cde76ec4fd 2024-11-15T22:11:35,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/b84685ef7f62405f8abf39cde76ec4fd, entries=150, sequenceid=76, filesize=11.7 K 2024-11-15T22:11:35,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/cb54fdd5050c4042a518e874fe8923d7 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/cb54fdd5050c4042a518e874fe8923d7 2024-11-15T22:11:35,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/cb54fdd5050c4042a518e874fe8923d7, entries=150, sequenceid=76, filesize=11.7 K 2024-11-15T22:11:35,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 3220684e1e7cfe8432a04d7ee5eae414 in 1213ms, sequenceid=76, compaction requested=false 2024-11-15T22:11:35,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:35,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T22:11:35,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:35,391 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 3220684e1e7cfe8432a04d7ee5eae414 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T22:11:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:35,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:35,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/e16124b28cb343e3bef99659956003ac is 50, key is test_row_0/A:col10/1731708694232/Put/seqid=0 2024-11-15T22:11:35,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741852_1028 (size=12001) 2024-11-15T22:11:35,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:35,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708755537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:35,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708755538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:35,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:35,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:35,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708755545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708755546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:35,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:35,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:35,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708755548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:35,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:35,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708755656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:35,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:35,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708755657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:35,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:35,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708755667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:35,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:35,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708755668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:35,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708755671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:35,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:35,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:35,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708755881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:35,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708755890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:35,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708755890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:35,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:35,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708755890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:35,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:35,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,897 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/e16124b28cb343e3bef99659956003ac 2024-11-15T22:11:35,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:35,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708755891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:35,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:35,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:35,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/a589a61a361648e1a2b8c324beb7c0b7 is 50, key is test_row_0/B:col10/1731708694232/Put/seqid=0 2024-11-15T22:11:35,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741853_1029 (size=12001) 2024-11-15T22:11:35,959 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/a589a61a361648e1a2b8c324beb7c0b7 2024-11-15T22:11:35,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/4fb0c02b9c274c9682b5fceb32cf2309 is 50, key is test_row_0/C:col10/1731708694232/Put/seqid=0 2024-11-15T22:11:36,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741854_1030 (size=12001) 2024-11-15T22:11:36,026 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/4fb0c02b9c274c9682b5fceb32cf2309 2024-11-15T22:11:36,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/e16124b28cb343e3bef99659956003ac as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/e16124b28cb343e3bef99659956003ac 2024-11-15T22:11:36,049 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/e16124b28cb343e3bef99659956003ac, entries=150, sequenceid=90, filesize=11.7 K 2024-11-15T22:11:36,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/a589a61a361648e1a2b8c324beb7c0b7 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/a589a61a361648e1a2b8c324beb7c0b7 2024-11-15T22:11:36,077 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/a589a61a361648e1a2b8c324beb7c0b7, entries=150, sequenceid=90, filesize=11.7 K 2024-11-15T22:11:36,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/4fb0c02b9c274c9682b5fceb32cf2309 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/4fb0c02b9c274c9682b5fceb32cf2309 2024-11-15T22:11:36,093 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/4fb0c02b9c274c9682b5fceb32cf2309, entries=150, sequenceid=90, filesize=11.7 K 2024-11-15T22:11:36,097 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-15T22:11:36,099 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-15T22:11:36,101 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3220684e1e7cfe8432a04d7ee5eae414 in 709ms, sequenceid=90, compaction requested=true 2024-11-15T22:11:36,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:36,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:36,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-15T22:11:36,102 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T22:11:36,102 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-15T22:11:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-15T22:11:36,102 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-15T22:11:36,102 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-15T22:11:36,104 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T22:11:36,104 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T22:11:36,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-15T22:11:36,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7110 sec 2024-11-15T22:11:36,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.7590 sec 2024-11-15T22:11:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:36,203 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:11:36,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3220684e1e7cfe8432a04d7ee5eae414 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T22:11:36,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/d41b47e657ae411d98e6521d3d99c97e is 50, key is test_row_0/A:col10/1731708695544/Put/seqid=0 2024-11-15T22:11:36,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:36,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708756221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:36,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:36,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:36,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708756223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:36,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:36,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:36,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708756227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:36,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:36,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:36,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708756229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:36,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:36,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708756230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:36,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741855_1031 (size=14341) 2024-11-15T22:11:36,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/d41b47e657ae411d98e6521d3d99c97e 2024-11-15T22:11:36,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/dde64703a9ba45ff84372ba20eba3e74 is 50, key is test_row_0/B:col10/1731708695544/Put/seqid=0 2024-11-15T22:11:36,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741856_1032 (size=12001) 2024-11-15T22:11:36,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708756338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:36,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:36,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:36,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:36,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708756338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:36,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:36,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708756338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708756338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:36,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708756349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:36,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:36,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708756549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:36,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T22:11:36,559 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:11:36,560 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:11:36,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-15T22:11:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708756558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:36,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:36,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708756559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T22:11:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:36,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:36,567 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:11:36,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708756560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708756561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:36,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:36,568 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:11:36,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:11:36,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T22:11:36,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T22:11:36,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:36,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:36,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:36,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:36,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:36,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/dde64703a9ba45ff84372ba20eba3e74 2024-11-15T22:11:36,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/1d15abbb719345139a949c401f8408bb is 50, key is test_row_0/C:col10/1731708695544/Put/seqid=0 2024-11-15T22:11:36,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741857_1033 (size=12001) 2024-11-15T22:11:36,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:36,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708756869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:36,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:36,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:36,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708756877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:36,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:36,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:36,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708756877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:36,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:36,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:36,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:36,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708756878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:36,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708756878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:36,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T22:11:36,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T22:11:36,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:36,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:36,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:36,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:36,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:36,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:37,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T22:11:37,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:37,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:37,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:37,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:37,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:37,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T22:11:37,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T22:11:37,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:37,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:37,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:37,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:37,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:37,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/1d15abbb719345139a949c401f8408bb 2024-11-15T22:11:37,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/d41b47e657ae411d98e6521d3d99c97e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d41b47e657ae411d98e6521d3d99c97e 2024-11-15T22:11:37,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d41b47e657ae411d98e6521d3d99c97e, entries=200, sequenceid=116, filesize=14.0 K 2024-11-15T22:11:37,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/dde64703a9ba45ff84372ba20eba3e74 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/dde64703a9ba45ff84372ba20eba3e74 2024-11-15T22:11:37,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/dde64703a9ba45ff84372ba20eba3e74, entries=150, sequenceid=116, filesize=11.7 K 2024-11-15T22:11:37,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/1d15abbb719345139a949c401f8408bb as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/1d15abbb719345139a949c401f8408bb 2024-11-15T22:11:37,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/1d15abbb719345139a949c401f8408bb, entries=150, sequenceid=116, filesize=11.7 K 2024-11-15T22:11:37,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 3220684e1e7cfe8432a04d7ee5eae414 in 1095ms, sequenceid=116, compaction requested=true 2024-11-15T22:11:37,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:37,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3220684e1e7cfe8432a04d7ee5eae414:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:11:37,299 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:11:37,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:37,299 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:11:37,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3220684e1e7cfe8432a04d7ee5eae414:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:11:37,302 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:11:37,303 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 3220684e1e7cfe8432a04d7ee5eae414/A is initiating minor compaction (all files) 2024-11-15T22:11:37,303 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3220684e1e7cfe8432a04d7ee5eae414/A in TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:37,303 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/043bb49612264f99bc5579e77792909b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/f199ca5c8e434bc7b5f0261444b3445f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/e16124b28cb343e3bef99659956003ac, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d41b47e657ae411d98e6521d3d99c97e] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp, totalSize=51.5 K 2024-11-15T22:11:37,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:37,305 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 043bb49612264f99bc5579e77792909b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731708693035 2024-11-15T22:11:37,305 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:11:37,305 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 3220684e1e7cfe8432a04d7ee5eae414/B is initiating minor compaction (all files) 2024-11-15T22:11:37,305 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3220684e1e7cfe8432a04d7ee5eae414/B in TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:37,306 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/babbd67dceeb4d67b17828eac2ffd4d8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/b84685ef7f62405f8abf39cde76ec4fd, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/a589a61a361648e1a2b8c324beb7c0b7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/dde64703a9ba45ff84372ba20eba3e74] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp, totalSize=47.0 K 2024-11-15T22:11:37,306 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting f199ca5c8e434bc7b5f0261444b3445f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731708693490 2024-11-15T22:11:37,308 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting babbd67dceeb4d67b17828eac2ffd4d8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731708693035 2024-11-15T22:11:37,308 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting e16124b28cb343e3bef99659956003ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731708694232 2024-11-15T22:11:37,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3220684e1e7cfe8432a04d7ee5eae414:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:11:37,309 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting b84685ef7f62405f8abf39cde76ec4fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731708693490 2024-11-15T22:11:37,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:11:37,310 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting a589a61a361648e1a2b8c324beb7c0b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731708694232 2024-11-15T22:11:37,311 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting dde64703a9ba45ff84372ba20eba3e74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731708695518 2024-11-15T22:11:37,316 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting d41b47e657ae411d98e6521d3d99c97e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731708695518 2024-11-15T22:11:37,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T22:11:37,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:37,375 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 3220684e1e7cfe8432a04d7ee5eae414 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-15T22:11:37,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:37,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/5e3b4a79f5e54dfea12d9ddfa76f82e9 is 50, key is test_row_0/A:col10/1731708696226/Put/seqid=0 2024-11-15T22:11:37,404 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3220684e1e7cfe8432a04d7ee5eae414#B#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:11:37,406 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/a8c06d66ec2f452098b2877796a936b6 is 50, key is test_row_0/B:col10/1731708695544/Put/seqid=0 2024-11-15T22:11:37,415 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3220684e1e7cfe8432a04d7ee5eae414#A#compaction#23 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:11:37,416 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/82fe4e6ca1474720a812b2790d1ca992 is 50, key is test_row_0/A:col10/1731708695544/Put/seqid=0 2024-11-15T22:11:37,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741858_1034 (size=12001) 2024-11-15T22:11:37,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741859_1035 (size=12241) 2024-11-15T22:11:37,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741860_1036 (size=12241) 2024-11-15T22:11:37,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708757471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:37,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708757475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:37,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708757477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:37,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708757483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708757489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:37,500 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/82fe4e6ca1474720a812b2790d1ca992 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/82fe4e6ca1474720a812b2790d1ca992 2024-11-15T22:11:37,505 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/a8c06d66ec2f452098b2877796a936b6 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/a8c06d66ec2f452098b2877796a936b6 2024-11-15T22:11:37,525 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3220684e1e7cfe8432a04d7ee5eae414/B of 3220684e1e7cfe8432a04d7ee5eae414 into a8c06d66ec2f452098b2877796a936b6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:11:37,528 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:37,528 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., storeName=3220684e1e7cfe8432a04d7ee5eae414/B, priority=12, startTime=1731708697299; duration=0sec 2024-11-15T22:11:37,528 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:11:37,528 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3220684e1e7cfe8432a04d7ee5eae414:B 2024-11-15T22:11:37,528 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:11:37,534 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:11:37,534 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 3220684e1e7cfe8432a04d7ee5eae414/C is initiating minor compaction (all files) 2024-11-15T22:11:37,534 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3220684e1e7cfe8432a04d7ee5eae414/C in TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:37,534 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/45189ca8d83347d398a3c1b9bca862a1, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/cb54fdd5050c4042a518e874fe8923d7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/4fb0c02b9c274c9682b5fceb32cf2309, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/1d15abbb719345139a949c401f8408bb] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp, totalSize=47.0 K 2024-11-15T22:11:37,536 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 45189ca8d83347d398a3c1b9bca862a1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731708693035 2024-11-15T22:11:37,536 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3220684e1e7cfe8432a04d7ee5eae414/A of 3220684e1e7cfe8432a04d7ee5eae414 into 82fe4e6ca1474720a812b2790d1ca992(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:11:37,536 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:37,536 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., storeName=3220684e1e7cfe8432a04d7ee5eae414/A, priority=12, startTime=1731708697299; duration=0sec 2024-11-15T22:11:37,537 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:37,537 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3220684e1e7cfe8432a04d7ee5eae414:A 2024-11-15T22:11:37,537 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting cb54fdd5050c4042a518e874fe8923d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731708693490 2024-11-15T22:11:37,538 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fb0c02b9c274c9682b5fceb32cf2309, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731708694232 2024-11-15T22:11:37,539 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d15abbb719345139a949c401f8408bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731708695518 2024-11-15T22:11:37,573 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3220684e1e7cfe8432a04d7ee5eae414#C#compaction#24 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:11:37,574 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/f450dd05d0614316a3be90569a3e552b is 50, key is test_row_0/C:col10/1731708695544/Put/seqid=0 2024-11-15T22:11:37,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:37,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708757597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708757597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708757598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:37,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:37,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:37,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708757598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:37,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708757599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:37,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:37,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:37,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741861_1037 (size=12241) 2024-11-15T22:11:37,669 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/f450dd05d0614316a3be90569a3e552b as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/f450dd05d0614316a3be90569a3e552b 2024-11-15T22:11:37,698 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3220684e1e7cfe8432a04d7ee5eae414/C of 3220684e1e7cfe8432a04d7ee5eae414 into f450dd05d0614316a3be90569a3e552b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:11:37,698 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:37,699 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., storeName=3220684e1e7cfe8432a04d7ee5eae414/C, priority=12, startTime=1731708697304; duration=0sec 2024-11-15T22:11:37,699 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:37,699 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3220684e1e7cfe8432a04d7ee5eae414:C 2024-11-15T22:11:37,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T22:11:37,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:37,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708757806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:37,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708757807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:37,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:37,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708757807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:37,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:37,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708757807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:37,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:37,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:37,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708757807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:37,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:37,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:37,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:37,852 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/5e3b4a79f5e54dfea12d9ddfa76f82e9 2024-11-15T22:11:37,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/7a2afc9028c743ba8cc39f46ea26452f is 50, key is test_row_0/B:col10/1731708696226/Put/seqid=0 2024-11-15T22:11:37,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741862_1038 (size=12001) 2024-11-15T22:11:38,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:38,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708758117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:38,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:38,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:38,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708758117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:38,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:38,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:38,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:38,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:38,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708758117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:38,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708758117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:38,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:38,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:38,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:38,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:38,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:38,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:38,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:38,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708758126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:38,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:38,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:38,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:38,344 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/7a2afc9028c743ba8cc39f46ea26452f 2024-11-15T22:11:38,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/1258eb7e9a7e40948f23a98f81e2a60c is 50, key is test_row_0/C:col10/1731708696226/Put/seqid=0 2024-11-15T22:11:38,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741863_1039 (size=12001) 2024-11-15T22:11:38,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708758627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:38,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:38,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:38,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:38,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708758627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:38,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:38,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:38,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:38,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:38,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708758633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:38,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:38,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:38,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:38,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:38,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708758633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:38,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:38,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:38,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:38,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708758638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:38,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:38,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:38,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:38,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T22:11:38,769 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/1258eb7e9a7e40948f23a98f81e2a60c 2024-11-15T22:11:38,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/5e3b4a79f5e54dfea12d9ddfa76f82e9 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/5e3b4a79f5e54dfea12d9ddfa76f82e9 2024-11-15T22:11:38,793 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/5e3b4a79f5e54dfea12d9ddfa76f82e9, entries=150, sequenceid=126, filesize=11.7 K 2024-11-15T22:11:38,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/7a2afc9028c743ba8cc39f46ea26452f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/7a2afc9028c743ba8cc39f46ea26452f 2024-11-15T22:11:38,810 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/7a2afc9028c743ba8cc39f46ea26452f, entries=150, sequenceid=126, filesize=11.7 K 2024-11-15T22:11:38,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/1258eb7e9a7e40948f23a98f81e2a60c as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/1258eb7e9a7e40948f23a98f81e2a60c 2024-11-15T22:11:38,832 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/1258eb7e9a7e40948f23a98f81e2a60c, entries=150, sequenceid=126, filesize=11.7 K 2024-11-15T22:11:38,836 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 3220684e1e7cfe8432a04d7ee5eae414 in 1460ms, sequenceid=126, compaction requested=false 2024-11-15T22:11:38,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:38,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:38,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-15T22:11:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-15T22:11:38,843 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-15T22:11:38,843 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2700 sec 2024-11-15T22:11:38,850 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.2850 sec 2024-11-15T22:11:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:39,649 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:11:39,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3220684e1e7cfe8432a04d7ee5eae414 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-15T22:11:39,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:39,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:39,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708759661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:39,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708759661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:39,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:39,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:39,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:39,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708759666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:39,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708759666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:39,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:39,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708759668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:39,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:39,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/c1f066f79e184cca97a41d7987bd30e7 is 50, key is test_row_0/A:col10/1731708699647/Put/seqid=0 2024-11-15T22:11:39,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741864_1040 (size=12151) 2024-11-15T22:11:39,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:39,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708759779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708759779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:39,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:39,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708759780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:39,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708759780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:39,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708759781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:39,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:39,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708759987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:39,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:39,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708759987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:39,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708759987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:39,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:39,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708759997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:40,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:40,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708759999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:40,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:40,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/c1f066f79e184cca97a41d7987bd30e7 2024-11-15T22:11:40,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/5dddcc5681074ec6abce7eb08500c40e is 50, key is test_row_0/B:col10/1731708699647/Put/seqid=0 2024-11-15T22:11:40,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741865_1041 (size=12151) 2024-11-15T22:11:40,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708760297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:40,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:40,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708760297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:40,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:40,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:40,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:40,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708760307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708760307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:40,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:40,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708760307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:40,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:40,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:40,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/5dddcc5681074ec6abce7eb08500c40e 2024-11-15T22:11:40,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/3abb7680a3fd410bb88283f119024e47 is 50, key is test_row_0/C:col10/1731708699647/Put/seqid=0 2024-11-15T22:11:40,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741866_1042 (size=12151) 2024-11-15T22:11:40,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/3abb7680a3fd410bb88283f119024e47 2024-11-15T22:11:40,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/c1f066f79e184cca97a41d7987bd30e7 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/c1f066f79e184cca97a41d7987bd30e7 2024-11-15T22:11:40,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/c1f066f79e184cca97a41d7987bd30e7, entries=150, sequenceid=157, filesize=11.9 K 2024-11-15T22:11:40,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/5dddcc5681074ec6abce7eb08500c40e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/5dddcc5681074ec6abce7eb08500c40e 2024-11-15T22:11:40,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/5dddcc5681074ec6abce7eb08500c40e, entries=150, sequenceid=157, filesize=11.9 K 2024-11-15T22:11:40,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/3abb7680a3fd410bb88283f119024e47 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/3abb7680a3fd410bb88283f119024e47 2024-11-15T22:11:40,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/3abb7680a3fd410bb88283f119024e47, entries=150, sequenceid=157, filesize=11.9 K 2024-11-15T22:11:40,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 3220684e1e7cfe8432a04d7ee5eae414 in 1078ms, sequenceid=157, compaction requested=true 2024-11-15T22:11:40,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:40,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3220684e1e7cfe8432a04d7ee5eae414:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:11:40,728 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:11:40,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:40,729 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:11:40,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3220684e1e7cfe8432a04d7ee5eae414:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:11:40,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:40,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3220684e1e7cfe8432a04d7ee5eae414:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:11:40,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:11:40,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T22:11:40,737 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:11:40,738 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:11:40,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-15T22:11:40,741 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:11:40,743 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:11:40,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:11:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T22:11:40,746 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:11:40,746 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 3220684e1e7cfe8432a04d7ee5eae414/A is initiating minor compaction (all files) 2024-11-15T22:11:40,746 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3220684e1e7cfe8432a04d7ee5eae414/A in TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:40,747 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:11:40,747 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 3220684e1e7cfe8432a04d7ee5eae414/B is initiating minor compaction (all files) 2024-11-15T22:11:40,747 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3220684e1e7cfe8432a04d7ee5eae414/B in TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:40,747 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/82fe4e6ca1474720a812b2790d1ca992, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/5e3b4a79f5e54dfea12d9ddfa76f82e9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/c1f066f79e184cca97a41d7987bd30e7] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp, totalSize=35.5 K 2024-11-15T22:11:40,747 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/a8c06d66ec2f452098b2877796a936b6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/7a2afc9028c743ba8cc39f46ea26452f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/5dddcc5681074ec6abce7eb08500c40e] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp, totalSize=35.5 K 2024-11-15T22:11:40,748 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82fe4e6ca1474720a812b2790d1ca992, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731708695518 2024-11-15T22:11:40,749 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting a8c06d66ec2f452098b2877796a936b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731708695518 2024-11-15T22:11:40,749 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e3b4a79f5e54dfea12d9ddfa76f82e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731708696209 2024-11-15T22:11:40,750 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a2afc9028c743ba8cc39f46ea26452f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731708696209 2024-11-15T22:11:40,750 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1f066f79e184cca97a41d7987bd30e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731708697474 2024-11-15T22:11:40,751 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dddcc5681074ec6abce7eb08500c40e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731708697474 2024-11-15T22:11:40,775 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3220684e1e7cfe8432a04d7ee5eae414#A#compaction#30 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:11:40,776 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/4c5486834c4348f68066c52efab88072 is 50, key is test_row_0/A:col10/1731708699647/Put/seqid=0 2024-11-15T22:11:40,794 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3220684e1e7cfe8432a04d7ee5eae414#B#compaction#31 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:11:40,795 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/5e34b4c2d3614866901ccdab46d4b97e is 50, key is test_row_0/B:col10/1731708699647/Put/seqid=0 2024-11-15T22:11:40,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741867_1043 (size=12493) 2024-11-15T22:11:40,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:40,827 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:11:40,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3220684e1e7cfe8432a04d7ee5eae414 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T22:11:40,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/fbece90985fb4411a04be301b4070ff9 is 50, key is test_row_0/A:col10/1731708700826/Put/seqid=0 2024-11-15T22:11:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T22:11:40,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741868_1044 (size=12493) 2024-11-15T22:11:40,865 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/5e34b4c2d3614866901ccdab46d4b97e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/5e34b4c2d3614866901ccdab46d4b97e 2024-11-15T22:11:40,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741869_1045 (size=12147) 2024-11-15T22:11:40,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/fbece90985fb4411a04be301b4070ff9 2024-11-15T22:11:40,877 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3220684e1e7cfe8432a04d7ee5eae414/B of 3220684e1e7cfe8432a04d7ee5eae414 into 5e34b4c2d3614866901ccdab46d4b97e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:11:40,877 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:40,878 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., storeName=3220684e1e7cfe8432a04d7ee5eae414/B, priority=13, startTime=1731708700728; duration=0sec 2024-11-15T22:11:40,878 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:11:40,878 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3220684e1e7cfe8432a04d7ee5eae414:B 2024-11-15T22:11:40,878 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:11:40,880 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:11:40,881 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 3220684e1e7cfe8432a04d7ee5eae414/C is initiating minor compaction (all files) 2024-11-15T22:11:40,881 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3220684e1e7cfe8432a04d7ee5eae414/C in TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:40,881 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/f450dd05d0614316a3be90569a3e552b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/1258eb7e9a7e40948f23a98f81e2a60c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/3abb7680a3fd410bb88283f119024e47] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp, totalSize=35.5 K 2024-11-15T22:11:40,882 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting f450dd05d0614316a3be90569a3e552b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731708695518 2024-11-15T22:11:40,883 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 1258eb7e9a7e40948f23a98f81e2a60c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731708696209 2024-11-15T22:11:40,884 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 3abb7680a3fd410bb88283f119024e47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731708697474 2024-11-15T22:11:40,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T22:11:40,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:40,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:40,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:40,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:40,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:40,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:40,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/988525c4fb944c93b8c075e29f53d2da is 50, key is test_row_0/B:col10/1731708700826/Put/seqid=0 2024-11-15T22:11:40,914 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3220684e1e7cfe8432a04d7ee5eae414#C#compaction#34 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:11:40,915 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/c8c51da9608f418aa9af421b5847c4d2 is 50, key is test_row_0/C:col10/1731708699647/Put/seqid=0 2024-11-15T22:11:40,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708760922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:40,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708760932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:40,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:40,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708760932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:40,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:40,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:40,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:40,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708760934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:40,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:40,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:40,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708760936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:40,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741870_1046 (size=12493) 2024-11-15T22:11:40,991 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/c8c51da9608f418aa9af421b5847c4d2 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/c8c51da9608f418aa9af421b5847c4d2 2024-11-15T22:11:40,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741871_1047 (size=9757) 2024-11-15T22:11:41,004 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3220684e1e7cfe8432a04d7ee5eae414/C of 3220684e1e7cfe8432a04d7ee5eae414 into c8c51da9608f418aa9af421b5847c4d2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:11:41,005 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:41,005 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., storeName=3220684e1e7cfe8432a04d7ee5eae414/C, priority=13, startTime=1731708700730; duration=0sec 2024-11-15T22:11:41,005 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:41,005 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3220684e1e7cfe8432a04d7ee5eae414:C 2024-11-15T22:11:41,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:41,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708761048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:41,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:41,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708761048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:41,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:41,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708761048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:41,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T22:11:41,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T22:11:41,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:41,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:41,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:41,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:41,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:41,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:41,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:41,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708761057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:41,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:41,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:41,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708761057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:41,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:41,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T22:11:41,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:41,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:41,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:41,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:41,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:41,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:41,257 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/4c5486834c4348f68066c52efab88072 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/4c5486834c4348f68066c52efab88072 2024-11-15T22:11:41,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:41,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708761257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:41,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:41,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:41,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708761258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:41,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:41,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:41,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708761258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:41,270 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3220684e1e7cfe8432a04d7ee5eae414/A of 3220684e1e7cfe8432a04d7ee5eae414 into 4c5486834c4348f68066c52efab88072(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:11:41,270 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:41,270 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., storeName=3220684e1e7cfe8432a04d7ee5eae414/A, priority=13, startTime=1731708700728; duration=0sec 2024-11-15T22:11:41,270 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:41,270 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3220684e1e7cfe8432a04d7ee5eae414:A 2024-11-15T22:11:41,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708761279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708761279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T22:11:41,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T22:11:41,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:41,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:41,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:41,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:41,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:41,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/988525c4fb944c93b8c075e29f53d2da 2024-11-15T22:11:41,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/ee9239ab78c94540b0a56ac8a866a390 is 50, key is test_row_0/C:col10/1731708700826/Put/seqid=0 2024-11-15T22:11:41,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741872_1048 (size=9757) 2024-11-15T22:11:41,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T22:11:41,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:41,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:41,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:41,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:41,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:41,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:41,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708761570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:41,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:41,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708761570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:41,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708761570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:41,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708761587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708761587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:41,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:41,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T22:11:41,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:41,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:41,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:41,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:41,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:41,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T22:11:41,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/ee9239ab78c94540b0a56ac8a866a390 2024-11-15T22:11:41,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T22:11:41,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:41,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:41,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:41,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:41,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:41,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:41,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/fbece90985fb4411a04be301b4070ff9 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/fbece90985fb4411a04be301b4070ff9 2024-11-15T22:11:41,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/fbece90985fb4411a04be301b4070ff9, entries=150, sequenceid=169, filesize=11.9 K 2024-11-15T22:11:41,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/988525c4fb944c93b8c075e29f53d2da as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/988525c4fb944c93b8c075e29f53d2da 2024-11-15T22:11:41,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/988525c4fb944c93b8c075e29f53d2da, entries=100, sequenceid=169, filesize=9.5 K 2024-11-15T22:11:41,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/ee9239ab78c94540b0a56ac8a866a390 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/ee9239ab78c94540b0a56ac8a866a390 2024-11-15T22:11:41,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/ee9239ab78c94540b0a56ac8a866a390, entries=100, sequenceid=169, filesize=9.5 K 2024-11-15T22:11:41,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3220684e1e7cfe8432a04d7ee5eae414 in 1147ms, sequenceid=169, compaction requested=false 2024-11-15T22:11:41,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:42,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T22:11:42,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:42,054 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 3220684e1e7cfe8432a04d7ee5eae414 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T22:11:42,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/7e1fc1d5ac184aae92cfb16aa41947c0 is 50, key is test_row_0/A:col10/1731708700869/Put/seqid=0 2024-11-15T22:11:42,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:42,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741873_1049 (size=12151) 2024-11-15T22:11:42,116 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/7e1fc1d5ac184aae92cfb16aa41947c0 2024-11-15T22:11:42,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708762107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:42,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:42,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708762116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:42,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:42,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708762121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:42,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:42,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708762122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:42,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:42,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:42,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708762124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:42,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:42,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/619fee0e9d6048b79a6e94d183aba32a is 50, key is test_row_0/B:col10/1731708700869/Put/seqid=0 2024-11-15T22:11:42,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741874_1050 (size=12151) 2024-11-15T22:11:42,185 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/619fee0e9d6048b79a6e94d183aba32a 2024-11-15T22:11:42,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/87c69c3b6488419e835248a270ce5143 is 50, key is test_row_0/C:col10/1731708700869/Put/seqid=0 2024-11-15T22:11:42,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741875_1051 (size=12151) 2024-11-15T22:11:42,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708762236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:42,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:42,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:42,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708762236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:42,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708762237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:42,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708762246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:42,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708762447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:42,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:42,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:42,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708762457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708762457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:42,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:42,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708762457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:42,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:42,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:42,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:42,639 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/87c69c3b6488419e835248a270ce5143 2024-11-15T22:11:42,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/7e1fc1d5ac184aae92cfb16aa41947c0 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/7e1fc1d5ac184aae92cfb16aa41947c0 2024-11-15T22:11:42,683 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/7e1fc1d5ac184aae92cfb16aa41947c0, entries=150, sequenceid=196, filesize=11.9 K 2024-11-15T22:11:42,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/619fee0e9d6048b79a6e94d183aba32a as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/619fee0e9d6048b79a6e94d183aba32a 2024-11-15T22:11:42,705 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/619fee0e9d6048b79a6e94d183aba32a, entries=150, sequenceid=196, filesize=11.9 K 2024-11-15T22:11:42,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/87c69c3b6488419e835248a270ce5143 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/87c69c3b6488419e835248a270ce5143 2024-11-15T22:11:42,731 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/87c69c3b6488419e835248a270ce5143, entries=150, sequenceid=196, filesize=11.9 K 2024-11-15T22:11:42,733 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 3220684e1e7cfe8432a04d7ee5eae414 in 679ms, sequenceid=196, compaction requested=true 2024-11-15T22:11:42,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:42,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:42,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-15T22:11:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-15T22:11:42,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-15T22:11:42,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9940 sec 2024-11-15T22:11:42,745 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.0050 sec 2024-11-15T22:11:42,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:42,775 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:11:42,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3220684e1e7cfe8432a04d7ee5eae414 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T22:11:42,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/d619cdd938bd4b6db31e709a6af2a2b9 is 50, key is test_row_0/A:col10/1731708702093/Put/seqid=0 2024-11-15T22:11:42,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741876_1052 (size=16931) 2024-11-15T22:11:42,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:42,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708762854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:42,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:42,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:42,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708762857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:42,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708762858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:42,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708762862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T22:11:42,897 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:11:42,898 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:11:42,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-15T22:11:42,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T22:11:42,902 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:11:42,903 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:11:42,904 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:11:42,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:42,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708762981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708762976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:42,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708762981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:42,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:42,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708762982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:42,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T22:11:43,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T22:11:43,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:43,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:43,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:43,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:43,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:43,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708763127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:43,133 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_2' in 3220684e1e7cfe8432a04d7ee5eae414 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:11:43,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:43,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:43,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:43,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:43,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708763197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708763197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:43,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:43,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708763197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708763197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T22:11:43,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T22:11:43,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:43,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:43,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:43,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:43,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:43,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/d619cdd938bd4b6db31e709a6af2a2b9 2024-11-15T22:11:43,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/ef656ec87ef949708dd8dbe91f4ee0b7 is 50, key is test_row_0/B:col10/1731708702093/Put/seqid=0 2024-11-15T22:11:43,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741877_1053 (size=12151) 2024-11-15T22:11:43,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T22:11:43,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:43,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:43,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:43,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:43,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:43,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:43,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:43,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708763506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:43,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:43,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708763507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:43,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:43,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708763507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:43,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:43,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:43,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:43,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:43,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708763517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:43,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:43,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:43,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T22:11:43,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T22:11:43,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:43,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:43,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:43,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:43,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:43,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:43,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/ef656ec87ef949708dd8dbe91f4ee0b7 2024-11-15T22:11:43,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/b2609e221ded4004b46fd04c75ed8367 is 50, key is test_row_0/C:col10/1731708702093/Put/seqid=0 2024-11-15T22:11:43,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T22:11:43,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:43,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:43,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:43,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:43,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:43,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:43,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741878_1054 (size=12151) 2024-11-15T22:11:43,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T22:11:43,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:43,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:43,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:43,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:43,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:43,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T22:11:43,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:43,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:43,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:43,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:43,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:43,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:44,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708764017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:44,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708764017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:44,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708764017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:44,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:44,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708764027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:44,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:44,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:44,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:44,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T22:11:44,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/b2609e221ded4004b46fd04c75ed8367 2024-11-15T22:11:44,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/d619cdd938bd4b6db31e709a6af2a2b9 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d619cdd938bd4b6db31e709a6af2a2b9 2024-11-15T22:11:44,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d619cdd938bd4b6db31e709a6af2a2b9, entries=250, sequenceid=211, filesize=16.5 K 2024-11-15T22:11:44,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/ef656ec87ef949708dd8dbe91f4ee0b7 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/ef656ec87ef949708dd8dbe91f4ee0b7 2024-11-15T22:11:44,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T22:11:44,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:44,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:44,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:44,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:44,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:44,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/ef656ec87ef949708dd8dbe91f4ee0b7, entries=150, sequenceid=211, filesize=11.9 K 2024-11-15T22:11:44,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/b2609e221ded4004b46fd04c75ed8367 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/b2609e221ded4004b46fd04c75ed8367 2024-11-15T22:11:44,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/b2609e221ded4004b46fd04c75ed8367, entries=150, sequenceid=211, filesize=11.9 K 2024-11-15T22:11:44,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 3220684e1e7cfe8432a04d7ee5eae414 in 1404ms, sequenceid=211, compaction requested=true 2024-11-15T22:11:44,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:44,180 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:11:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3220684e1e7cfe8432a04d7ee5eae414:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:11:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:44,181 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:11:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3220684e1e7cfe8432a04d7ee5eae414:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:11:44,183 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53722 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:11:44,183 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 3220684e1e7cfe8432a04d7ee5eae414/A is initiating minor compaction (all files) 2024-11-15T22:11:44,183 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3220684e1e7cfe8432a04d7ee5eae414/A in TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:44,183 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/4c5486834c4348f68066c52efab88072, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/fbece90985fb4411a04be301b4070ff9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/7e1fc1d5ac184aae92cfb16aa41947c0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d619cdd938bd4b6db31e709a6af2a2b9] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp, totalSize=52.5 K 2024-11-15T22:11:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:44,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3220684e1e7cfe8432a04d7ee5eae414:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:11:44,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:11:44,185 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:11:44,185 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 3220684e1e7cfe8432a04d7ee5eae414/B is initiating minor compaction (all files) 2024-11-15T22:11:44,185 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3220684e1e7cfe8432a04d7ee5eae414/B in TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:44,185 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/5e34b4c2d3614866901ccdab46d4b97e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/988525c4fb944c93b8c075e29f53d2da, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/619fee0e9d6048b79a6e94d183aba32a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/ef656ec87ef949708dd8dbe91f4ee0b7] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp, totalSize=45.5 K 2024-11-15T22:11:44,186 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e34b4c2d3614866901ccdab46d4b97e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731708697474 2024-11-15T22:11:44,186 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c5486834c4348f68066c52efab88072, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731708697474 2024-11-15T22:11:44,187 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 988525c4fb944c93b8c075e29f53d2da, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731708700824 2024-11-15T22:11:44,187 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbece90985fb4411a04be301b4070ff9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731708700823 2024-11-15T22:11:44,187 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 619fee0e9d6048b79a6e94d183aba32a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731708700869 2024-11-15T22:11:44,188 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e1fc1d5ac184aae92cfb16aa41947c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731708700869 2024-11-15T22:11:44,188 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting ef656ec87ef949708dd8dbe91f4ee0b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731708702093 2024-11-15T22:11:44,189 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting d619cdd938bd4b6db31e709a6af2a2b9, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731708702093 2024-11-15T22:11:44,213 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3220684e1e7cfe8432a04d7ee5eae414#B#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:11:44,214 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/fcf5a2daa8674203b97aba4a46c079ff is 50, key is test_row_0/B:col10/1731708702093/Put/seqid=0 2024-11-15T22:11:44,218 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3220684e1e7cfe8432a04d7ee5eae414#A#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:11:44,218 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/5c5d1875d39d4a8aaf9f117e758f1c91 is 50, key is test_row_0/A:col10/1731708702093/Put/seqid=0 2024-11-15T22:11:44,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741879_1055 (size=12629) 2024-11-15T22:11:44,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741880_1056 (size=12629) 2024-11-15T22:11:44,240 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/fcf5a2daa8674203b97aba4a46c079ff as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/fcf5a2daa8674203b97aba4a46c079ff 2024-11-15T22:11:44,252 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3220684e1e7cfe8432a04d7ee5eae414/B of 3220684e1e7cfe8432a04d7ee5eae414 into fcf5a2daa8674203b97aba4a46c079ff(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:11:44,252 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:44,252 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., storeName=3220684e1e7cfe8432a04d7ee5eae414/B, priority=12, startTime=1731708704181; duration=0sec 2024-11-15T22:11:44,252 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:11:44,252 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3220684e1e7cfe8432a04d7ee5eae414:B 2024-11-15T22:11:44,252 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:11:44,260 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:11:44,260 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 3220684e1e7cfe8432a04d7ee5eae414/C is initiating minor compaction (all files) 2024-11-15T22:11:44,260 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3220684e1e7cfe8432a04d7ee5eae414/C in TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:44,260 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/c8c51da9608f418aa9af421b5847c4d2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/ee9239ab78c94540b0a56ac8a866a390, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/87c69c3b6488419e835248a270ce5143, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/b2609e221ded4004b46fd04c75ed8367] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp, totalSize=45.5 K 2024-11-15T22:11:44,261 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting c8c51da9608f418aa9af421b5847c4d2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731708697474 2024-11-15T22:11:44,262 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting ee9239ab78c94540b0a56ac8a866a390, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731708700824 2024-11-15T22:11:44,263 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 87c69c3b6488419e835248a270ce5143, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731708700869 2024-11-15T22:11:44,263 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting b2609e221ded4004b46fd04c75ed8367, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731708702093 2024-11-15T22:11:44,278 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3220684e1e7cfe8432a04d7ee5eae414#C#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:11:44,279 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/39fd00d9cbca4380b4112e3e8cc9d0a3 is 50, key is test_row_0/C:col10/1731708702093/Put/seqid=0 2024-11-15T22:11:44,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741881_1057 (size=12629) 2024-11-15T22:11:44,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T22:11:44,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:44,312 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 3220684e1e7cfe8432a04d7ee5eae414 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T22:11:44,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/fd3ae5e78ebc42cf988bf5df7a20c42d is 50, key is test_row_0/A:col10/1731708702855/Put/seqid=0 2024-11-15T22:11:44,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741882_1058 (size=12151) 2024-11-15T22:11:44,336 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/fd3ae5e78ebc42cf988bf5df7a20c42d 2024-11-15T22:11:44,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/1452298ddaab4da9b0250f84db059d5b is 50, key is test_row_0/B:col10/1731708702855/Put/seqid=0 2024-11-15T22:11:44,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741883_1059 (size=12151) 2024-11-15T22:11:44,370 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/1452298ddaab4da9b0250f84db059d5b 2024-11-15T22:11:44,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/62e2918b1dde4c11ab1a4d8c1ba299a4 is 50, key is test_row_0/C:col10/1731708702855/Put/seqid=0 2024-11-15T22:11:44,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741884_1060 (size=12151) 2024-11-15T22:11:44,405 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/62e2918b1dde4c11ab1a4d8c1ba299a4 2024-11-15T22:11:44,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/fd3ae5e78ebc42cf988bf5df7a20c42d as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/fd3ae5e78ebc42cf988bf5df7a20c42d 2024-11-15T22:11:44,428 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/fd3ae5e78ebc42cf988bf5df7a20c42d, entries=150, sequenceid=234, filesize=11.9 K 2024-11-15T22:11:44,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/1452298ddaab4da9b0250f84db059d5b as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/1452298ddaab4da9b0250f84db059d5b 2024-11-15T22:11:44,443 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/1452298ddaab4da9b0250f84db059d5b, entries=150, sequenceid=234, filesize=11.9 K 2024-11-15T22:11:44,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/62e2918b1dde4c11ab1a4d8c1ba299a4 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/62e2918b1dde4c11ab1a4d8c1ba299a4 2024-11-15T22:11:44,463 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/62e2918b1dde4c11ab1a4d8c1ba299a4, entries=150, sequenceid=234, filesize=11.9 K 2024-11-15T22:11:44,464 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 3220684e1e7cfe8432a04d7ee5eae414 in 153ms, sequenceid=234, compaction requested=false 2024-11-15T22:11:44,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:44,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:44,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-15T22:11:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-15T22:11:44,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-15T22:11:44,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5640 sec 2024-11-15T22:11:44,475 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.5740 sec 2024-11-15T22:11:44,647 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/5c5d1875d39d4a8aaf9f117e758f1c91 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/5c5d1875d39d4a8aaf9f117e758f1c91 2024-11-15T22:11:44,665 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3220684e1e7cfe8432a04d7ee5eae414/A of 3220684e1e7cfe8432a04d7ee5eae414 into 5c5d1875d39d4a8aaf9f117e758f1c91(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:11:44,665 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:44,665 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., storeName=3220684e1e7cfe8432a04d7ee5eae414/A, priority=12, startTime=1731708704180; duration=0sec 2024-11-15T22:11:44,666 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:44,666 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3220684e1e7cfe8432a04d7ee5eae414:A 2024-11-15T22:11:44,707 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/39fd00d9cbca4380b4112e3e8cc9d0a3 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/39fd00d9cbca4380b4112e3e8cc9d0a3 2024-11-15T22:11:44,718 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3220684e1e7cfe8432a04d7ee5eae414/C of 3220684e1e7cfe8432a04d7ee5eae414 into 39fd00d9cbca4380b4112e3e8cc9d0a3(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:11:44,718 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:44,718 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., storeName=3220684e1e7cfe8432a04d7ee5eae414/C, priority=12, startTime=1731708704184; duration=0sec 2024-11-15T22:11:44,718 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:44,718 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3220684e1e7cfe8432a04d7ee5eae414:C 2024-11-15T22:11:45,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T22:11:45,048 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:11:45,050 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:11:45,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-15T22:11:45,057 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:11:45,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T22:11:45,058 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:11:45,058 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:11:45,070 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:11:45,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3220684e1e7cfe8432a04d7ee5eae414 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T22:11:45,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:45,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/79e6f2238c8845d2bfe46c2fa2220039 is 50, key is test_row_0/A:col10/1731708705063/Put/seqid=0 2024-11-15T22:11:45,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741885_1061 (size=14537) 2024-11-15T22:11:45,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T22:11:45,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708765195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:45,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:45,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708765196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:45,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:45,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708765197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:45,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:45,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:45,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T22:11:45,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:45,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:45,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:45,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:45,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:45,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:45,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:45,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708765208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:45,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:45,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:45,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708765204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:45,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:45,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708765317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:45,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:45,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:45,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708765317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:45,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:45,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:45,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708765317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:45,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708765327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:45,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708765327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:45,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T22:11:45,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:45,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:45,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:45,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:45,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:45,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T22:11:45,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T22:11:45,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:45,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:45,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:45,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:45,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:45,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708765527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708765527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:45,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708765527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:45,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:45,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708765536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:45,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:45,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708765537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:45,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:45,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:45,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/79e6f2238c8845d2bfe46c2fa2220039 2024-11-15T22:11:45,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/2f639ac412e0480da328828571834162 is 50, key is test_row_0/B:col10/1731708705063/Put/seqid=0 2024-11-15T22:11:45,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741886_1062 (size=9757) 2024-11-15T22:11:45,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T22:11:45,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:45,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:45,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:45,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:45,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T22:11:45,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T22:11:45,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:45,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:45,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:45,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:45,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:45,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708765836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:45,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:45,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708765847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:45,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:45,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708765847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708765847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:45,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708765847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:45,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:45,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:45,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:45,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T22:11:45,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:45,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:45,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:45,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:45,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:45,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:45,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/2f639ac412e0480da328828571834162 2024-11-15T22:11:46,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/9ed7481c481242a9826ef865bc5aadda is 50, key is test_row_0/C:col10/1731708705063/Put/seqid=0 2024-11-15T22:11:46,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741887_1063 (size=9757) 2024-11-15T22:11:46,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/9ed7481c481242a9826ef865bc5aadda 2024-11-15T22:11:46,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/79e6f2238c8845d2bfe46c2fa2220039 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/79e6f2238c8845d2bfe46c2fa2220039 2024-11-15T22:11:46,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/79e6f2238c8845d2bfe46c2fa2220039, entries=200, sequenceid=247, filesize=14.2 K 2024-11-15T22:11:46,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/2f639ac412e0480da328828571834162 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/2f639ac412e0480da328828571834162 2024-11-15T22:11:46,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/2f639ac412e0480da328828571834162, entries=100, sequenceid=247, filesize=9.5 K 2024-11-15T22:11:46,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/9ed7481c481242a9826ef865bc5aadda as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/9ed7481c481242a9826ef865bc5aadda 2024-11-15T22:11:46,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/9ed7481c481242a9826ef865bc5aadda, entries=100, sequenceid=247, filesize=9.5 K 2024-11-15T22:11:46,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3220684e1e7cfe8432a04d7ee5eae414 in 1062ms, sequenceid=247, compaction requested=true 2024-11-15T22:11:46,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:46,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3220684e1e7cfe8432a04d7ee5eae414:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:11:46,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:46,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3220684e1e7cfe8432a04d7ee5eae414:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:11:46,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:46,132 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:11:46,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3220684e1e7cfe8432a04d7ee5eae414:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:11:46,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:11:46,134 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:11:46,134 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 3220684e1e7cfe8432a04d7ee5eae414/A is initiating minor compaction (all files) 2024-11-15T22:11:46,134 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3220684e1e7cfe8432a04d7ee5eae414/A in TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:46,134 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/5c5d1875d39d4a8aaf9f117e758f1c91, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/fd3ae5e78ebc42cf988bf5df7a20c42d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/79e6f2238c8845d2bfe46c2fa2220039] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp, totalSize=38.4 K 2024-11-15T22:11:46,136 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c5d1875d39d4a8aaf9f117e758f1c91, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731708702093 2024-11-15T22:11:46,132 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:11:46,142 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd3ae5e78ebc42cf988bf5df7a20c42d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731708702851 2024-11-15T22:11:46,143 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79e6f2238c8845d2bfe46c2fa2220039, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731708705055 2024-11-15T22:11:46,144 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:11:46,146 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 3220684e1e7cfe8432a04d7ee5eae414/B is initiating minor compaction (all files) 2024-11-15T22:11:46,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T22:11:46,146 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3220684e1e7cfe8432a04d7ee5eae414/B in TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:46,147 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/fcf5a2daa8674203b97aba4a46c079ff, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/1452298ddaab4da9b0250f84db059d5b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/2f639ac412e0480da328828571834162] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp, totalSize=33.7 K 2024-11-15T22:11:46,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:46,147 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 3220684e1e7cfe8432a04d7ee5eae414 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T22:11:46,149 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting fcf5a2daa8674203b97aba4a46c079ff, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731708702093 2024-11-15T22:11:46,152 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 1452298ddaab4da9b0250f84db059d5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731708702851 2024-11-15T22:11:46,153 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f639ac412e0480da328828571834162, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731708705063 2024-11-15T22:11:46,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/a46fd75d167241819e19f09112e4f414 is 50, key is test_row_0/A:col10/1731708705184/Put/seqid=0 2024-11-15T22:11:46,166 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3220684e1e7cfe8432a04d7ee5eae414#B#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:11:46,167 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/4795aeb4d0014301ae757ecf9767ea63 is 50, key is test_row_0/B:col10/1731708705063/Put/seqid=0 2024-11-15T22:11:46,172 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3220684e1e7cfe8432a04d7ee5eae414#A#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:11:46,172 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/dae7363ca25b4633938705337ca7385a is 50, key is test_row_0/A:col10/1731708705063/Put/seqid=0 2024-11-15T22:11:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T22:11:46,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741888_1064 (size=12301) 2024-11-15T22:11:46,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741889_1065 (size=12731) 2024-11-15T22:11:46,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741890_1066 (size=12731) 2024-11-15T22:11:46,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:46,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:46,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708766392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:46,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708766392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:46,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708766392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:46,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:46,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708766394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:46,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:46,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:46,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708766398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:46,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708766507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:46,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708766507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:46,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:46,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708766508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708766509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708766509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:46,615 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/a46fd75d167241819e19f09112e4f414 2024-11-15T22:11:46,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/f71c4cf65c534a41a6d045cd8e5fbc76 is 50, key is test_row_0/B:col10/1731708705184/Put/seqid=0 2024-11-15T22:11:46,651 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/4795aeb4d0014301ae757ecf9767ea63 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/4795aeb4d0014301ae757ecf9767ea63 2024-11-15T22:11:46,654 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/dae7363ca25b4633938705337ca7385a as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/dae7363ca25b4633938705337ca7385a 2024-11-15T22:11:46,668 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3220684e1e7cfe8432a04d7ee5eae414/A of 3220684e1e7cfe8432a04d7ee5eae414 into dae7363ca25b4633938705337ca7385a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:11:46,668 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:46,668 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., storeName=3220684e1e7cfe8432a04d7ee5eae414/A, priority=13, startTime=1731708706132; duration=0sec 2024-11-15T22:11:46,668 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:11:46,668 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3220684e1e7cfe8432a04d7ee5eae414:A 2024-11-15T22:11:46,668 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:11:46,669 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3220684e1e7cfe8432a04d7ee5eae414/B of 3220684e1e7cfe8432a04d7ee5eae414 into 4795aeb4d0014301ae757ecf9767ea63(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:11:46,670 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:46,670 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., storeName=3220684e1e7cfe8432a04d7ee5eae414/B, priority=13, startTime=1731708706132; duration=0sec 2024-11-15T22:11:46,670 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:11:46,670 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:46,670 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 3220684e1e7cfe8432a04d7ee5eae414/C is initiating minor compaction (all files) 2024-11-15T22:11:46,670 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3220684e1e7cfe8432a04d7ee5eae414:B 2024-11-15T22:11:46,670 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3220684e1e7cfe8432a04d7ee5eae414/C in TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:46,670 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/39fd00d9cbca4380b4112e3e8cc9d0a3, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/62e2918b1dde4c11ab1a4d8c1ba299a4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/9ed7481c481242a9826ef865bc5aadda] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp, totalSize=33.7 K 2024-11-15T22:11:46,671 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39fd00d9cbca4380b4112e3e8cc9d0a3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731708702093 2024-11-15T22:11:46,671 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62e2918b1dde4c11ab1a4d8c1ba299a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731708702851 2024-11-15T22:11:46,671 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ed7481c481242a9826ef865bc5aadda, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731708705063 2024-11-15T22:11:46,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741891_1067 (size=12301) 2024-11-15T22:11:46,683 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3220684e1e7cfe8432a04d7ee5eae414#C#compaction#55 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:11:46,683 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/7e3ee07f323d48b5b3aeed0fc5385374 is 50, key is test_row_0/C:col10/1731708705063/Put/seqid=0 2024-11-15T22:11:46,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741892_1068 (size=12731) 2024-11-15T22:11:46,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708766717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:46,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708766717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:46,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708766717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:46,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708766717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:46,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:46,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708766717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:47,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:47,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:47,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708767027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:47,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708767027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:47,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708767027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:47,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:47,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:47,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708767027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:47,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708767027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:47,081 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/f71c4cf65c534a41a6d045cd8e5fbc76 2024-11-15T22:11:47,116 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/7e3ee07f323d48b5b3aeed0fc5385374 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/7e3ee07f323d48b5b3aeed0fc5385374 2024-11-15T22:11:47,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/61912aeff1c7430a9f3965d1eae94ee3 is 50, key is test_row_0/C:col10/1731708705184/Put/seqid=0 2024-11-15T22:11:47,140 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3220684e1e7cfe8432a04d7ee5eae414/C of 3220684e1e7cfe8432a04d7ee5eae414 into 7e3ee07f323d48b5b3aeed0fc5385374(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:11:47,140 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:47,140 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., storeName=3220684e1e7cfe8432a04d7ee5eae414/C, priority=13, startTime=1731708706132; duration=0sec 2024-11-15T22:11:47,141 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:47,141 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3220684e1e7cfe8432a04d7ee5eae414:C 2024-11-15T22:11:47,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741893_1069 (size=12301) 2024-11-15T22:11:47,147 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/61912aeff1c7430a9f3965d1eae94ee3 2024-11-15T22:11:47,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/a46fd75d167241819e19f09112e4f414 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/a46fd75d167241819e19f09112e4f414 2024-11-15T22:11:47,164 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/a46fd75d167241819e19f09112e4f414, entries=150, sequenceid=272, filesize=12.0 K 2024-11-15T22:11:47,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/f71c4cf65c534a41a6d045cd8e5fbc76 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/f71c4cf65c534a41a6d045cd8e5fbc76 2024-11-15T22:11:47,179 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/f71c4cf65c534a41a6d045cd8e5fbc76, entries=150, sequenceid=272, filesize=12.0 K 2024-11-15T22:11:47,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/61912aeff1c7430a9f3965d1eae94ee3 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/61912aeff1c7430a9f3965d1eae94ee3 2024-11-15T22:11:47,194 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/61912aeff1c7430a9f3965d1eae94ee3, entries=150, sequenceid=272, filesize=12.0 K 2024-11-15T22:11:47,195 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 3220684e1e7cfe8432a04d7ee5eae414 in 1048ms, sequenceid=272, compaction requested=false 2024-11-15T22:11:47,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:47,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:47,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-15T22:11:47,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-15T22:11:47,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-15T22:11:47,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1400 sec 2024-11-15T22:11:47,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.1530 sec 2024-11-15T22:11:47,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T22:11:47,207 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:11:47,208 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:11:47,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-15T22:11:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T22:11:47,212 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:11:47,213 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:11:47,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:11:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T22:11:47,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T22:11:47,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:47,375 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 3220684e1e7cfe8432a04d7ee5eae414 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T22:11:47,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/afd7ffa27e99444eb6cc499b64c64d0c is 50, key is test_row_0/A:col10/1731708706375/Put/seqid=0 2024-11-15T22:11:47,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741894_1070 (size=12301) 2024-11-15T22:11:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T22:11:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:47,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:47,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708767603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:47,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:47,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708767603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:47,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:47,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708767604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:47,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708767608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:47,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708767608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:47,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708767730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:47,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708767730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:47,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:47,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708767731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:47,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:47,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708767731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:47,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708767731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:47,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:47,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:47,825 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/afd7ffa27e99444eb6cc499b64c64d0c 2024-11-15T22:11:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T22:11:47,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/ba521cb03739404692e6476f842f2d4d is 50, key is test_row_0/B:col10/1731708706375/Put/seqid=0 2024-11-15T22:11:47,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741895_1071 (size=12301) 2024-11-15T22:11:47,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708767947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:47,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:47,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708767947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:47,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:47,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708767948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:47,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:47,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708767948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:47,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:47,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708767948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:47,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:47,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:48,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708768258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:48,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708768258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:48,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708768258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:48,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708768267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:48,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708768267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:48,323 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/ba521cb03739404692e6476f842f2d4d 2024-11-15T22:11:48,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/9b2c04c2ef90484283ce9b1ffdfc95a0 is 50, key is test_row_0/C:col10/1731708706375/Put/seqid=0 2024-11-15T22:11:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T22:11:48,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741896_1072 (size=12301) 2024-11-15T22:11:48,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708768767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:48,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708768767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:48,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:48,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708768767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:48,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:48,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:48,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:48,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:48,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:48,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:48,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708768777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:48,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708768787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:48,814 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/9b2c04c2ef90484283ce9b1ffdfc95a0 2024-11-15T22:11:48,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/afd7ffa27e99444eb6cc499b64c64d0c as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/afd7ffa27e99444eb6cc499b64c64d0c 2024-11-15T22:11:48,845 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/afd7ffa27e99444eb6cc499b64c64d0c, entries=150, sequenceid=286, filesize=12.0 K 2024-11-15T22:11:48,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/ba521cb03739404692e6476f842f2d4d as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/ba521cb03739404692e6476f842f2d4d 2024-11-15T22:11:48,869 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/ba521cb03739404692e6476f842f2d4d, entries=150, sequenceid=286, filesize=12.0 K 2024-11-15T22:11:48,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/9b2c04c2ef90484283ce9b1ffdfc95a0 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/9b2c04c2ef90484283ce9b1ffdfc95a0 2024-11-15T22:11:48,880 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/9b2c04c2ef90484283ce9b1ffdfc95a0, entries=150, sequenceid=286, filesize=12.0 K 2024-11-15T22:11:48,881 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 3220684e1e7cfe8432a04d7ee5eae414 in 1507ms, sequenceid=286, compaction requested=true 2024-11-15T22:11:48,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:48,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:48,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-15T22:11:48,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-15T22:11:48,887 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-15T22:11:48,887 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6710 sec 2024-11-15T22:11:48,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.6800 sec 2024-11-15T22:11:49,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T22:11:49,367 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:11:49,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:11:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-15T22:11:49,370 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:11:49,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T22:11:49,371 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:11:49,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:11:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T22:11:49,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T22:11:49,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:49,528 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 3220684e1e7cfe8432a04d7ee5eae414 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T22:11:49,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/a1013bb58a144acda4484d397fa3bf8e is 50, key is test_row_0/A:col10/1731708707605/Put/seqid=0 2024-11-15T22:11:49,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741897_1073 (size=12301) 2024-11-15T22:11:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T22:11:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:49,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:49,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708769806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:49,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708769807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:49,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708769808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:49,812 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_1' in 3220684e1e7cfe8432a04d7ee5eae414 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:11:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:49,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:49,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708769810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:49,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:49,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:49,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:49,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:49,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708769811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:49,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:49,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:49,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:49,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708769917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:49,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708769917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:49,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:49,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708769926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:49,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:49,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:49,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:49,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708769926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:49,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:49,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:49,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:49,943 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/a1013bb58a144acda4484d397fa3bf8e 2024-11-15T22:11:49,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/0eebdfe55e134e2fa5034fd440193959 is 50, key is test_row_0/B:col10/1731708707605/Put/seqid=0 2024-11-15T22:11:49,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741898_1074 (size=12301) 2024-11-15T22:11:49,978 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/0eebdfe55e134e2fa5034fd440193959 2024-11-15T22:11:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T22:11:50,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/292b37f8a2624690a67250b7fc4ef6b6 is 50, key is test_row_0/C:col10/1731708707605/Put/seqid=0 2024-11-15T22:11:50,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741899_1075 (size=12301) 2024-11-15T22:11:50,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:50,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708770126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:50,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:50,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708770127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:50,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:50,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:50,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:50,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708770137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:50,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708770137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:50,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:50,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:50,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708770436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:50,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:50,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708770436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:50,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:50,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:50,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:50,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:50,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:50,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:50,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:50,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708770447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:50,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:50,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708770447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:50,452 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/292b37f8a2624690a67250b7fc4ef6b6 2024-11-15T22:11:50,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/a1013bb58a144acda4484d397fa3bf8e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/a1013bb58a144acda4484d397fa3bf8e 2024-11-15T22:11:50,469 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/a1013bb58a144acda4484d397fa3bf8e, entries=150, sequenceid=312, filesize=12.0 K 2024-11-15T22:11:50,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/0eebdfe55e134e2fa5034fd440193959 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/0eebdfe55e134e2fa5034fd440193959 2024-11-15T22:11:50,479 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/0eebdfe55e134e2fa5034fd440193959, entries=150, sequenceid=312, filesize=12.0 K 2024-11-15T22:11:50,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/292b37f8a2624690a67250b7fc4ef6b6 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/292b37f8a2624690a67250b7fc4ef6b6 2024-11-15T22:11:50,504 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/292b37f8a2624690a67250b7fc4ef6b6, entries=150, sequenceid=312, filesize=12.0 K 2024-11-15T22:11:50,506 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 3220684e1e7cfe8432a04d7ee5eae414 in 978ms, sequenceid=312, compaction requested=true 2024-11-15T22:11:50,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:50,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T22:11:50,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-15T22:11:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-15T22:11:50,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-15T22:11:50,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1420 sec 2024-11-15T22:11:50,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.1490 sec 2024-11-15T22:11:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:50,952 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:11:50,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3220684e1e7cfe8432a04d7ee5eae414 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T22:11:50,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/2db4d9ea6e5346998ef6de5012726f33 is 50, key is test_row_0/A:col10/1731708709808/Put/seqid=0 2024-11-15T22:11:50,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741900_1076 (size=12301) 2024-11-15T22:11:50,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/2db4d9ea6e5346998ef6de5012726f33 2024-11-15T22:11:51,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/bde5d84b6d214abdb92b1017bde39264 is 50, key is test_row_0/B:col10/1731708709808/Put/seqid=0 2024-11-15T22:11:51,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741901_1077 (size=12301) 2024-11-15T22:11:51,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:51,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708771053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708771055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:51,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708771057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:51,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708771057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:51,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708771170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:51,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708771170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:51,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:51,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708771170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:51,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708771170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:51,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:51,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708771387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708771387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708771387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:51,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708771388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:51,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:51,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:51,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/bde5d84b6d214abdb92b1017bde39264 2024-11-15T22:11:51,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/db9d0c5c91db4e1aa0fd4e5c56b96e1f is 50, key is test_row_0/C:col10/1731708709808/Put/seqid=0 2024-11-15T22:11:51,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741902_1078 (size=12301) 2024-11-15T22:11:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T22:11:51,516 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:11:51,518 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:11:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-15T22:11:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T22:11:51,524 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:11:51,525 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:11:51,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:11:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T22:11:51,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T22:11:51,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:51,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:51,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:51,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:51,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:51,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:51,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708771697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708771697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:51,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708771698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708771698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:51,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54612 deadline: 1731708771826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:51,828 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_1' in 3220684e1e7cfe8432a04d7ee5eae414 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:11:51,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:51,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:51,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T22:11:51,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T22:11:51,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:51,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:51,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:51,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:11:51,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/db9d0c5c91db4e1aa0fd4e5c56b96e1f 2024-11-15T22:11:51,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/2db4d9ea6e5346998ef6de5012726f33 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/2db4d9ea6e5346998ef6de5012726f33 2024-11-15T22:11:51,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/2db4d9ea6e5346998ef6de5012726f33, entries=150, sequenceid=324, filesize=12.0 K 2024-11-15T22:11:51,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/bde5d84b6d214abdb92b1017bde39264 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/bde5d84b6d214abdb92b1017bde39264 2024-11-15T22:11:51,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/bde5d84b6d214abdb92b1017bde39264, entries=150, sequenceid=324, filesize=12.0 K 2024-11-15T22:11:51,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/db9d0c5c91db4e1aa0fd4e5c56b96e1f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/db9d0c5c91db4e1aa0fd4e5c56b96e1f 2024-11-15T22:11:51,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/db9d0c5c91db4e1aa0fd4e5c56b96e1f, entries=150, sequenceid=324, filesize=12.0 K 2024-11-15T22:11:51,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3220684e1e7cfe8432a04d7ee5eae414 in 994ms, sequenceid=324, compaction requested=true 2024-11-15T22:11:51,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:51,953 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T22:11:51,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3220684e1e7cfe8432a04d7ee5eae414:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:11:51,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:51,954 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T22:11:51,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3220684e1e7cfe8432a04d7ee5eae414:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:11:51,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:51,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3220684e1e7cfe8432a04d7ee5eae414:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:11:51,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:11:51,968 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61935 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T22:11:51,968 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 3220684e1e7cfe8432a04d7ee5eae414/A is initiating minor compaction (all files) 2024-11-15T22:11:51,968 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3220684e1e7cfe8432a04d7ee5eae414/A in TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:51,968 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/dae7363ca25b4633938705337ca7385a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/a46fd75d167241819e19f09112e4f414, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/afd7ffa27e99444eb6cc499b64c64d0c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/a1013bb58a144acda4484d397fa3bf8e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/2db4d9ea6e5346998ef6de5012726f33] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp, totalSize=60.5 K 2024-11-15T22:11:51,969 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61935 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T22:11:51,969 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 3220684e1e7cfe8432a04d7ee5eae414/B is initiating minor compaction (all files) 2024-11-15T22:11:51,969 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3220684e1e7cfe8432a04d7ee5eae414/B in TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:51,969 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/4795aeb4d0014301ae757ecf9767ea63, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/f71c4cf65c534a41a6d045cd8e5fbc76, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/ba521cb03739404692e6476f842f2d4d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/0eebdfe55e134e2fa5034fd440193959, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/bde5d84b6d214abdb92b1017bde39264] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp, totalSize=60.5 K 2024-11-15T22:11:51,974 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting dae7363ca25b4633938705337ca7385a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731708702854 2024-11-15T22:11:51,974 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting a46fd75d167241819e19f09112e4f414, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731708705184 2024-11-15T22:11:51,975 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 4795aeb4d0014301ae757ecf9767ea63, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731708702854 2024-11-15T22:11:51,975 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting afd7ffa27e99444eb6cc499b64c64d0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731708706375 2024-11-15T22:11:51,975 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting f71c4cf65c534a41a6d045cd8e5fbc76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731708705184 2024-11-15T22:11:51,975 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1013bb58a144acda4484d397fa3bf8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731708707592 2024-11-15T22:11:51,976 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting ba521cb03739404692e6476f842f2d4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731708706375 2024-11-15T22:11:51,976 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2db4d9ea6e5346998ef6de5012726f33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731708709802 2024-11-15T22:11:51,977 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eebdfe55e134e2fa5034fd440193959, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731708707592 2024-11-15T22:11:51,977 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting bde5d84b6d214abdb92b1017bde39264, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731708709802 2024-11-15T22:11:51,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T22:11:51,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:51,990 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 3220684e1e7cfe8432a04d7ee5eae414 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T22:11:52,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/d844d359c77d45c98ea3e1f77756015b is 50, key is test_row_0/A:col10/1731708711053/Put/seqid=0 2024-11-15T22:11:52,018 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3220684e1e7cfe8432a04d7ee5eae414#B#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:11:52,019 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/28752cf11a324898a70e11e0731576f9 is 50, key is test_row_0/B:col10/1731708709808/Put/seqid=0 2024-11-15T22:11:52,028 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3220684e1e7cfe8432a04d7ee5eae414#A#compaction#68 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:11:52,029 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/d3b3814c5d91430d981227708bf653a2 is 50, key is test_row_0/A:col10/1731708709808/Put/seqid=0 2024-11-15T22:11:52,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741903_1079 (size=12301) 2024-11-15T22:11:52,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741904_1080 (size=13051) 2024-11-15T22:11:52,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741905_1081 (size=13051) 2024-11-15T22:11:52,090 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/28752cf11a324898a70e11e0731576f9 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/28752cf11a324898a70e11e0731576f9 2024-11-15T22:11:52,102 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 3220684e1e7cfe8432a04d7ee5eae414/B of 3220684e1e7cfe8432a04d7ee5eae414 into 28752cf11a324898a70e11e0731576f9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:11:52,102 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:52,102 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., storeName=3220684e1e7cfe8432a04d7ee5eae414/B, priority=11, startTime=1731708711953; duration=0sec 2024-11-15T22:11:52,102 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:11:52,102 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3220684e1e7cfe8432a04d7ee5eae414:B 2024-11-15T22:11:52,102 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T22:11:52,107 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61935 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T22:11:52,107 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 3220684e1e7cfe8432a04d7ee5eae414/C is initiating minor compaction (all files) 2024-11-15T22:11:52,107 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3220684e1e7cfe8432a04d7ee5eae414/C in TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:52,107 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/7e3ee07f323d48b5b3aeed0fc5385374, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/61912aeff1c7430a9f3965d1eae94ee3, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/9b2c04c2ef90484283ce9b1ffdfc95a0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/292b37f8a2624690a67250b7fc4ef6b6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/db9d0c5c91db4e1aa0fd4e5c56b96e1f] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp, totalSize=60.5 K 2024-11-15T22:11:52,108 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e3ee07f323d48b5b3aeed0fc5385374, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731708702854 2024-11-15T22:11:52,109 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 61912aeff1c7430a9f3965d1eae94ee3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731708705184 2024-11-15T22:11:52,111 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b2c04c2ef90484283ce9b1ffdfc95a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731708706375 2024-11-15T22:11:52,111 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 292b37f8a2624690a67250b7fc4ef6b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731708707592 2024-11-15T22:11:52,112 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting db9d0c5c91db4e1aa0fd4e5c56b96e1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731708709802 2024-11-15T22:11:52,137 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3220684e1e7cfe8432a04d7ee5eae414#C#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:11:52,138 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/15d69b296e1f48ae97c2b4ffaa43086e is 50, key is test_row_0/C:col10/1731708709808/Put/seqid=0 2024-11-15T22:11:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T22:11:52,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741906_1082 (size=13051) 2024-11-15T22:11:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:52,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. as already flushing 2024-11-15T22:11:52,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708772261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:52,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708772262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:52,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:52,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708772279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:52,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:52,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708772285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:52,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:52,366 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-15T22:11:52,367 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:11:52,368 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-15T22:11:52,368 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:11:52,369 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-15T22:11:52,369 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:11:52,370 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-15T22:11:52,370 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:11:52,375 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:52,375 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:52,377 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:52,377 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:52,377 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:11:52,377 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:11:52,377 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:52,378 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:52,378 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:11:52,378 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:52,378 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:52,378 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:11:52,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:52,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708772386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708772386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:52,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:52,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:52,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708772396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:52,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:52,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708772407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:52,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:52,445 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/d844d359c77d45c98ea3e1f77756015b 2024-11-15T22:11:52,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/354e293a929144178bd8161b2ec24621 is 50, key is test_row_0/B:col10/1731708711053/Put/seqid=0 2024-11-15T22:11:52,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741907_1083 (size=12301) 2024-11-15T22:11:52,486 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/354e293a929144178bd8161b2ec24621 2024-11-15T22:11:52,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/e150c0c912b14488a589cb6a6f429ddc is 50, key is test_row_0/C:col10/1731708711053/Put/seqid=0 2024-11-15T22:11:52,506 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/d3b3814c5d91430d981227708bf653a2 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d3b3814c5d91430d981227708bf653a2 2024-11-15T22:11:52,528 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 3220684e1e7cfe8432a04d7ee5eae414/A of 3220684e1e7cfe8432a04d7ee5eae414 into d3b3814c5d91430d981227708bf653a2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:11:52,528 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:52,528 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., storeName=3220684e1e7cfe8432a04d7ee5eae414/A, priority=11, startTime=1731708711953; duration=0sec 2024-11-15T22:11:52,528 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:52,528 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3220684e1e7cfe8432a04d7ee5eae414:A 2024-11-15T22:11:52,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741908_1084 (size=12301) 2024-11-15T22:11:52,573 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/15d69b296e1f48ae97c2b4ffaa43086e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/15d69b296e1f48ae97c2b4ffaa43086e 2024-11-15T22:11:52,581 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 3220684e1e7cfe8432a04d7ee5eae414/C of 3220684e1e7cfe8432a04d7ee5eae414 into 15d69b296e1f48ae97c2b4ffaa43086e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:11:52,581 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:52,581 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., storeName=3220684e1e7cfe8432a04d7ee5eae414/C, priority=11, startTime=1731708711954; duration=0sec 2024-11-15T22:11:52,581 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:52,581 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3220684e1e7cfe8432a04d7ee5eae414:C 2024-11-15T22:11:52,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:52,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708772597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708772597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:52,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:52,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:52,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708772606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:52,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:52,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708772616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T22:11:52,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:52,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54598 deadline: 1731708772906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54596 deadline: 1731708772906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:52,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:52,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:52,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54614 deadline: 1731708772916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:52,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:11:52,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54584 deadline: 1731708772927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:52,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:11:52,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:11:52,951 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/e150c0c912b14488a589cb6a6f429ddc 2024-11-15T22:11:52,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/d844d359c77d45c98ea3e1f77756015b as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d844d359c77d45c98ea3e1f77756015b 2024-11-15T22:11:52,962 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d844d359c77d45c98ea3e1f77756015b, entries=150, sequenceid=348, filesize=12.0 K 2024-11-15T22:11:52,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/354e293a929144178bd8161b2ec24621 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/354e293a929144178bd8161b2ec24621 2024-11-15T22:11:52,968 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/354e293a929144178bd8161b2ec24621, entries=150, sequenceid=348, filesize=12.0 K 2024-11-15T22:11:52,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/e150c0c912b14488a589cb6a6f429ddc as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/e150c0c912b14488a589cb6a6f429ddc 2024-11-15T22:11:52,973 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/e150c0c912b14488a589cb6a6f429ddc, entries=150, sequenceid=348, filesize=12.0 K 2024-11-15T22:11:52,974 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 3220684e1e7cfe8432a04d7ee5eae414 in 984ms, sequenceid=348, compaction requested=false 2024-11-15T22:11:52,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:52,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:52,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-15T22:11:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-15T22:11:52,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-15T22:11:52,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4510 sec 2024-11-15T22:11:52,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.4610 sec 2024-11-15T22:11:53,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:53,419 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:11:53,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3220684e1e7cfe8432a04d7ee5eae414 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T22:11:53,419 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-15T22:11:53,419 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-15T22:11:53,419 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:11:53,419 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:53,419 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:11:53,419 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:53,420 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:53,420 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:53,420 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:11:53,420 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:11:53,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/9c7f0d78b3d24dc6bb8e5d9c75e1ca95 is 50, key is test_row_0/A:col10/1731708713417/Put/seqid=0 2024-11-15T22:11:53,428 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-15T22:11:53,428 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:11:53,428 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:53,428 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:53,428 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:11:53,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741909_1085 (size=12301) 2024-11-15T22:11:53,438 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-15T22:11:53,438 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:11:53,438 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:53,438 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:53,439 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:11:53,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T22:11:53,667 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:11:53,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/9c7f0d78b3d24dc6bb8e5d9c75e1ca95 2024-11-15T22:11:53,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/b23cfc5c80ea454d8eba2ad9b91ce3c4 is 50, key is test_row_0/B:col10/1731708713417/Put/seqid=0 2024-11-15T22:11:53,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741910_1086 (size=12301) 2024-11-15T22:11:54,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/b23cfc5c80ea454d8eba2ad9b91ce3c4 2024-11-15T22:11:54,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/eed82c067e7a4cc3abbc95f7fd44eb2b is 50, key is test_row_0/C:col10/1731708713417/Put/seqid=0 2024-11-15T22:11:54,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741911_1087 (size=12301) 2024-11-15T22:11:54,573 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T22:11:54,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/eed82c067e7a4cc3abbc95f7fd44eb2b 2024-11-15T22:11:54,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/9c7f0d78b3d24dc6bb8e5d9c75e1ca95 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/9c7f0d78b3d24dc6bb8e5d9c75e1ca95 2024-11-15T22:11:54,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/9c7f0d78b3d24dc6bb8e5d9c75e1ca95, entries=150, sequenceid=366, filesize=12.0 K 2024-11-15T22:11:54,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/b23cfc5c80ea454d8eba2ad9b91ce3c4 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/b23cfc5c80ea454d8eba2ad9b91ce3c4 2024-11-15T22:11:54,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/b23cfc5c80ea454d8eba2ad9b91ce3c4, entries=150, sequenceid=366, filesize=12.0 K 2024-11-15T22:11:54,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/eed82c067e7a4cc3abbc95f7fd44eb2b as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/eed82c067e7a4cc3abbc95f7fd44eb2b 2024-11-15T22:11:54,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/eed82c067e7a4cc3abbc95f7fd44eb2b, entries=150, sequenceid=366, filesize=12.0 K 2024-11-15T22:11:54,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=13.42 KB/13740 for 3220684e1e7cfe8432a04d7ee5eae414 in 1273ms, sequenceid=366, compaction requested=true 2024-11-15T22:11:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3220684e1e7cfe8432a04d7ee5eae414:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:11:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:54,692 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:11:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3220684e1e7cfe8432a04d7ee5eae414:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:11:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3220684e1e7cfe8432a04d7ee5eae414:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:11:54,692 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:11:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:11:54,694 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:11:54,694 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:11:54,694 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 3220684e1e7cfe8432a04d7ee5eae414/B is initiating minor compaction (all files) 2024-11-15T22:11:54,694 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 3220684e1e7cfe8432a04d7ee5eae414/A is initiating minor compaction (all files) 2024-11-15T22:11:54,694 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3220684e1e7cfe8432a04d7ee5eae414/B in TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:54,694 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3220684e1e7cfe8432a04d7ee5eae414/A in TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:54,694 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/28752cf11a324898a70e11e0731576f9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/354e293a929144178bd8161b2ec24621, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/b23cfc5c80ea454d8eba2ad9b91ce3c4] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp, totalSize=36.8 K 2024-11-15T22:11:54,694 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d3b3814c5d91430d981227708bf653a2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d844d359c77d45c98ea3e1f77756015b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/9c7f0d78b3d24dc6bb8e5d9c75e1ca95] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp, totalSize=36.8 K 2024-11-15T22:11:54,695 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 28752cf11a324898a70e11e0731576f9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731708709802 2024-11-15T22:11:54,695 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3b3814c5d91430d981227708bf653a2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731708709802 2024-11-15T22:11:54,695 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 354e293a929144178bd8161b2ec24621, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731708711053 2024-11-15T22:11:54,695 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting d844d359c77d45c98ea3e1f77756015b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731708711053 2024-11-15T22:11:54,696 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c7f0d78b3d24dc6bb8e5d9c75e1ca95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731708712261 2024-11-15T22:11:54,696 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting b23cfc5c80ea454d8eba2ad9b91ce3c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731708712261 2024-11-15T22:11:54,712 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3220684e1e7cfe8432a04d7ee5eae414#B#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:11:54,713 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/1e5c92d2bd2d4fe2b0f8e3debb2774e3 is 50, key is test_row_0/B:col10/1731708713417/Put/seqid=0 2024-11-15T22:11:54,715 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3220684e1e7cfe8432a04d7ee5eae414#A#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:11:54,716 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/034cc55db7604df3aa48409ca24d7f4a is 50, key is test_row_0/A:col10/1731708713417/Put/seqid=0 2024-11-15T22:11:54,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741912_1088 (size=13153) 2024-11-15T22:11:54,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741913_1089 (size=13153) 2024-11-15T22:11:54,730 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/034cc55db7604df3aa48409ca24d7f4a as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/034cc55db7604df3aa48409ca24d7f4a 2024-11-15T22:11:54,736 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3220684e1e7cfe8432a04d7ee5eae414/A of 3220684e1e7cfe8432a04d7ee5eae414 into 034cc55db7604df3aa48409ca24d7f4a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:11:54,736 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:54,736 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., storeName=3220684e1e7cfe8432a04d7ee5eae414/A, priority=13, startTime=1731708714692; duration=0sec 2024-11-15T22:11:54,736 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:11:54,736 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3220684e1e7cfe8432a04d7ee5eae414:A 2024-11-15T22:11:54,736 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:11:54,737 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:11:54,737 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 3220684e1e7cfe8432a04d7ee5eae414/C is initiating minor compaction (all files) 2024-11-15T22:11:54,737 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3220684e1e7cfe8432a04d7ee5eae414/C in TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:54,737 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/15d69b296e1f48ae97c2b4ffaa43086e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/e150c0c912b14488a589cb6a6f429ddc, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/eed82c067e7a4cc3abbc95f7fd44eb2b] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp, totalSize=36.8 K 2024-11-15T22:11:54,738 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15d69b296e1f48ae97c2b4ffaa43086e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731708709802 2024-11-15T22:11:54,738 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting e150c0c912b14488a589cb6a6f429ddc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731708711053 2024-11-15T22:11:54,739 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting eed82c067e7a4cc3abbc95f7fd44eb2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731708712261 2024-11-15T22:11:54,749 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3220684e1e7cfe8432a04d7ee5eae414#C#compaction#77 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:11:54,750 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/ed4979ebe0554e6aaa3472251f2ab0df is 50, key is test_row_0/C:col10/1731708713417/Put/seqid=0 2024-11-15T22:11:54,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741914_1090 (size=13153) 2024-11-15T22:11:55,126 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/1e5c92d2bd2d4fe2b0f8e3debb2774e3 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/1e5c92d2bd2d4fe2b0f8e3debb2774e3 2024-11-15T22:11:55,132 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3220684e1e7cfe8432a04d7ee5eae414/B of 3220684e1e7cfe8432a04d7ee5eae414 into 1e5c92d2bd2d4fe2b0f8e3debb2774e3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:11:55,133 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:55,133 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., storeName=3220684e1e7cfe8432a04d7ee5eae414/B, priority=13, startTime=1731708714692; duration=0sec 2024-11-15T22:11:55,133 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:55,133 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3220684e1e7cfe8432a04d7ee5eae414:B 2024-11-15T22:11:55,162 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/ed4979ebe0554e6aaa3472251f2ab0df as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/ed4979ebe0554e6aaa3472251f2ab0df 2024-11-15T22:11:55,167 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3220684e1e7cfe8432a04d7ee5eae414/C of 3220684e1e7cfe8432a04d7ee5eae414 into ed4979ebe0554e6aaa3472251f2ab0df(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:11:55,167 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3220684e1e7cfe8432a04d7ee5eae414: 2024-11-15T22:11:55,168 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., storeName=3220684e1e7cfe8432a04d7ee5eae414/C, priority=13, startTime=1731708714692; duration=0sec 2024-11-15T22:11:55,168 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:11:55,168 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3220684e1e7cfe8432a04d7ee5eae414:C 2024-11-15T22:11:55,879 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-15T22:11:55,879 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:11:55,879 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:55,879 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:55,879 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:11:55,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T22:11:55,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-15T22:11:55,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-15T22:11:55,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-15T22:11:55,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-15T22:11:55,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-15T22:11:55,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T22:11:55,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2456 2024-11-15T22:11:55,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2524 2024-11-15T22:11:55,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T22:11:55,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1198 2024-11-15T22:11:55,880 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3593 rows 2024-11-15T22:11:55,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1202 2024-11-15T22:11:55,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3606 rows 2024-11-15T22:11:55,881 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T22:11:55,881 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:11:55,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:55,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:11:55,885 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:11:55,886 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T22:11:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T22:11:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T22:11:55,896 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708715895"}]},"ts":"1731708715895"} 2024-11-15T22:11:55,898 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T22:11:55,898 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T22:11:55,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T22:11:55,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3220684e1e7cfe8432a04d7ee5eae414, UNASSIGN}] 2024-11-15T22:11:55,908 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3220684e1e7cfe8432a04d7ee5eae414, UNASSIGN 2024-11-15T22:11:55,910 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=3220684e1e7cfe8432a04d7ee5eae414, regionState=CLOSING, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:55,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3220684e1e7cfe8432a04d7ee5eae414, UNASSIGN because future has completed 2024-11-15T22:11:55,912 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T22:11:55,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516}] 2024-11-15T22:11:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T22:11:56,070 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:56,070 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T22:11:56,070 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing 3220684e1e7cfe8432a04d7ee5eae414, disabling compactions & flushes 2024-11-15T22:11:56,070 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:56,070 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:56,070 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. after waiting 0 ms 2024-11-15T22:11:56,071 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:56,071 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing 3220684e1e7cfe8432a04d7ee5eae414 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-15T22:11:56,075 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/74b99484c0254d5f97b6d801e4d46d2a is 50, key is test_row_1/A:col10/1731708715877/Put/seqid=0 2024-11-15T22:11:56,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741915_1091 (size=7415) 2024-11-15T22:11:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T22:11:56,480 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/74b99484c0254d5f97b6d801e4d46d2a 2024-11-15T22:11:56,490 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/8a800bf056674b5d920a17d61ad35606 is 50, key is test_row_1/B:col10/1731708715877/Put/seqid=0 2024-11-15T22:11:56,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741916_1092 (size=7415) 2024-11-15T22:11:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T22:11:56,895 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/8a800bf056674b5d920a17d61ad35606 2024-11-15T22:11:56,903 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/98d305ed599940f98f1492ea5a0024a3 is 50, key is test_row_1/C:col10/1731708715877/Put/seqid=0 2024-11-15T22:11:56,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741917_1093 (size=7415) 2024-11-15T22:11:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T22:11:57,308 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/98d305ed599940f98f1492ea5a0024a3 2024-11-15T22:11:57,315 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/A/74b99484c0254d5f97b6d801e4d46d2a as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/74b99484c0254d5f97b6d801e4d46d2a 2024-11-15T22:11:57,321 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/74b99484c0254d5f97b6d801e4d46d2a, entries=50, sequenceid=375, filesize=7.2 K 2024-11-15T22:11:57,322 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/B/8a800bf056674b5d920a17d61ad35606 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/8a800bf056674b5d920a17d61ad35606 2024-11-15T22:11:57,330 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/8a800bf056674b5d920a17d61ad35606, entries=50, sequenceid=375, filesize=7.2 K 2024-11-15T22:11:57,332 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/.tmp/C/98d305ed599940f98f1492ea5a0024a3 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/98d305ed599940f98f1492ea5a0024a3 2024-11-15T22:11:57,337 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/98d305ed599940f98f1492ea5a0024a3, entries=50, sequenceid=375, filesize=7.2 K 2024-11-15T22:11:57,338 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 3220684e1e7cfe8432a04d7ee5eae414 in 1267ms, sequenceid=375, compaction requested=false 2024-11-15T22:11:57,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/bd8754ef173e40229c2fe87f0349e03e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/741434a0e7f44bcb826fffad8632a7e0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/a3006ef71f2145b2bcc639aa836d35f1, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/043bb49612264f99bc5579e77792909b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/f199ca5c8e434bc7b5f0261444b3445f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/e16124b28cb343e3bef99659956003ac, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d41b47e657ae411d98e6521d3d99c97e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/82fe4e6ca1474720a812b2790d1ca992, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/5e3b4a79f5e54dfea12d9ddfa76f82e9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/4c5486834c4348f68066c52efab88072, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/c1f066f79e184cca97a41d7987bd30e7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/fbece90985fb4411a04be301b4070ff9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/7e1fc1d5ac184aae92cfb16aa41947c0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d619cdd938bd4b6db31e709a6af2a2b9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/5c5d1875d39d4a8aaf9f117e758f1c91, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/fd3ae5e78ebc42cf988bf5df7a20c42d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/79e6f2238c8845d2bfe46c2fa2220039, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/dae7363ca25b4633938705337ca7385a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/a46fd75d167241819e19f09112e4f414, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/afd7ffa27e99444eb6cc499b64c64d0c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/a1013bb58a144acda4484d397fa3bf8e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d3b3814c5d91430d981227708bf653a2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/2db4d9ea6e5346998ef6de5012726f33, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d844d359c77d45c98ea3e1f77756015b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/9c7f0d78b3d24dc6bb8e5d9c75e1ca95] to archive 2024-11-15T22:11:57,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:11:57,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/bd8754ef173e40229c2fe87f0349e03e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/bd8754ef173e40229c2fe87f0349e03e 2024-11-15T22:11:57,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/741434a0e7f44bcb826fffad8632a7e0 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/741434a0e7f44bcb826fffad8632a7e0 2024-11-15T22:11:57,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/a3006ef71f2145b2bcc639aa836d35f1 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/a3006ef71f2145b2bcc639aa836d35f1 2024-11-15T22:11:57,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/043bb49612264f99bc5579e77792909b to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/043bb49612264f99bc5579e77792909b 2024-11-15T22:11:57,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/f199ca5c8e434bc7b5f0261444b3445f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/f199ca5c8e434bc7b5f0261444b3445f 2024-11-15T22:11:57,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/e16124b28cb343e3bef99659956003ac to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/e16124b28cb343e3bef99659956003ac 2024-11-15T22:11:57,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d41b47e657ae411d98e6521d3d99c97e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d41b47e657ae411d98e6521d3d99c97e 2024-11-15T22:11:57,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/82fe4e6ca1474720a812b2790d1ca992 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/82fe4e6ca1474720a812b2790d1ca992 2024-11-15T22:11:57,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/5e3b4a79f5e54dfea12d9ddfa76f82e9 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/5e3b4a79f5e54dfea12d9ddfa76f82e9 2024-11-15T22:11:57,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/4c5486834c4348f68066c52efab88072 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/4c5486834c4348f68066c52efab88072 2024-11-15T22:11:57,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/c1f066f79e184cca97a41d7987bd30e7 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/c1f066f79e184cca97a41d7987bd30e7 2024-11-15T22:11:57,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/fbece90985fb4411a04be301b4070ff9 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/fbece90985fb4411a04be301b4070ff9 2024-11-15T22:11:57,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/7e1fc1d5ac184aae92cfb16aa41947c0 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/7e1fc1d5ac184aae92cfb16aa41947c0 2024-11-15T22:11:57,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d619cdd938bd4b6db31e709a6af2a2b9 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d619cdd938bd4b6db31e709a6af2a2b9 2024-11-15T22:11:57,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/5c5d1875d39d4a8aaf9f117e758f1c91 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/5c5d1875d39d4a8aaf9f117e758f1c91 2024-11-15T22:11:57,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/fd3ae5e78ebc42cf988bf5df7a20c42d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/fd3ae5e78ebc42cf988bf5df7a20c42d 2024-11-15T22:11:57,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/79e6f2238c8845d2bfe46c2fa2220039 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/79e6f2238c8845d2bfe46c2fa2220039 2024-11-15T22:11:57,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/dae7363ca25b4633938705337ca7385a to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/dae7363ca25b4633938705337ca7385a 2024-11-15T22:11:57,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/a46fd75d167241819e19f09112e4f414 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/a46fd75d167241819e19f09112e4f414 2024-11-15T22:11:57,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/afd7ffa27e99444eb6cc499b64c64d0c to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/afd7ffa27e99444eb6cc499b64c64d0c 2024-11-15T22:11:57,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/a1013bb58a144acda4484d397fa3bf8e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/a1013bb58a144acda4484d397fa3bf8e 2024-11-15T22:11:57,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d3b3814c5d91430d981227708bf653a2 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d3b3814c5d91430d981227708bf653a2 2024-11-15T22:11:57,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/2db4d9ea6e5346998ef6de5012726f33 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/2db4d9ea6e5346998ef6de5012726f33 2024-11-15T22:11:57,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d844d359c77d45c98ea3e1f77756015b to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/d844d359c77d45c98ea3e1f77756015b 2024-11-15T22:11:57,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/9c7f0d78b3d24dc6bb8e5d9c75e1ca95 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/9c7f0d78b3d24dc6bb8e5d9c75e1ca95 2024-11-15T22:11:57,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/b09c4538f2bc43f3b01fb985716f333d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/f5d4d428df354d99858aad10492887b0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/babbd67dceeb4d67b17828eac2ffd4d8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/cb0855ad35304ecfb1dee5c5a50b9a23, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/b84685ef7f62405f8abf39cde76ec4fd, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/a589a61a361648e1a2b8c324beb7c0b7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/a8c06d66ec2f452098b2877796a936b6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/dde64703a9ba45ff84372ba20eba3e74, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/7a2afc9028c743ba8cc39f46ea26452f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/5e34b4c2d3614866901ccdab46d4b97e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/5dddcc5681074ec6abce7eb08500c40e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/988525c4fb944c93b8c075e29f53d2da, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/619fee0e9d6048b79a6e94d183aba32a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/fcf5a2daa8674203b97aba4a46c079ff, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/ef656ec87ef949708dd8dbe91f4ee0b7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/1452298ddaab4da9b0250f84db059d5b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/4795aeb4d0014301ae757ecf9767ea63, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/2f639ac412e0480da328828571834162, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/f71c4cf65c534a41a6d045cd8e5fbc76, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/ba521cb03739404692e6476f842f2d4d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/0eebdfe55e134e2fa5034fd440193959, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/28752cf11a324898a70e11e0731576f9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/bde5d84b6d214abdb92b1017bde39264, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/354e293a929144178bd8161b2ec24621, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/b23cfc5c80ea454d8eba2ad9b91ce3c4] to archive 2024-11-15T22:11:57,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:11:57,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/b09c4538f2bc43f3b01fb985716f333d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/b09c4538f2bc43f3b01fb985716f333d 2024-11-15T22:11:57,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/f5d4d428df354d99858aad10492887b0 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/f5d4d428df354d99858aad10492887b0 2024-11-15T22:11:57,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/babbd67dceeb4d67b17828eac2ffd4d8 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/babbd67dceeb4d67b17828eac2ffd4d8 2024-11-15T22:11:57,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/cb0855ad35304ecfb1dee5c5a50b9a23 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/cb0855ad35304ecfb1dee5c5a50b9a23 2024-11-15T22:11:57,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/b84685ef7f62405f8abf39cde76ec4fd to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/b84685ef7f62405f8abf39cde76ec4fd 2024-11-15T22:11:57,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/a589a61a361648e1a2b8c324beb7c0b7 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/a589a61a361648e1a2b8c324beb7c0b7 2024-11-15T22:11:57,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/a8c06d66ec2f452098b2877796a936b6 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/a8c06d66ec2f452098b2877796a936b6 2024-11-15T22:11:57,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/dde64703a9ba45ff84372ba20eba3e74 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/dde64703a9ba45ff84372ba20eba3e74 2024-11-15T22:11:57,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/7a2afc9028c743ba8cc39f46ea26452f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/7a2afc9028c743ba8cc39f46ea26452f 2024-11-15T22:11:57,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/5e34b4c2d3614866901ccdab46d4b97e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/5e34b4c2d3614866901ccdab46d4b97e 2024-11-15T22:11:57,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/5dddcc5681074ec6abce7eb08500c40e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/5dddcc5681074ec6abce7eb08500c40e 2024-11-15T22:11:57,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/988525c4fb944c93b8c075e29f53d2da to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/988525c4fb944c93b8c075e29f53d2da 2024-11-15T22:11:57,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/619fee0e9d6048b79a6e94d183aba32a to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/619fee0e9d6048b79a6e94d183aba32a 2024-11-15T22:11:57,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/fcf5a2daa8674203b97aba4a46c079ff to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/fcf5a2daa8674203b97aba4a46c079ff 2024-11-15T22:11:57,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/ef656ec87ef949708dd8dbe91f4ee0b7 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/ef656ec87ef949708dd8dbe91f4ee0b7 2024-11-15T22:11:57,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/1452298ddaab4da9b0250f84db059d5b to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/1452298ddaab4da9b0250f84db059d5b 2024-11-15T22:11:57,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/4795aeb4d0014301ae757ecf9767ea63 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/4795aeb4d0014301ae757ecf9767ea63 2024-11-15T22:11:57,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/2f639ac412e0480da328828571834162 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/2f639ac412e0480da328828571834162 2024-11-15T22:11:57,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/f71c4cf65c534a41a6d045cd8e5fbc76 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/f71c4cf65c534a41a6d045cd8e5fbc76 2024-11-15T22:11:57,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/ba521cb03739404692e6476f842f2d4d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/ba521cb03739404692e6476f842f2d4d 2024-11-15T22:11:57,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/0eebdfe55e134e2fa5034fd440193959 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/0eebdfe55e134e2fa5034fd440193959 2024-11-15T22:11:57,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/28752cf11a324898a70e11e0731576f9 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/28752cf11a324898a70e11e0731576f9 2024-11-15T22:11:57,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/bde5d84b6d214abdb92b1017bde39264 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/bde5d84b6d214abdb92b1017bde39264 2024-11-15T22:11:57,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/354e293a929144178bd8161b2ec24621 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/354e293a929144178bd8161b2ec24621 2024-11-15T22:11:57,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/b23cfc5c80ea454d8eba2ad9b91ce3c4 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/b23cfc5c80ea454d8eba2ad9b91ce3c4 2024-11-15T22:11:57,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/baf192ea83744b11a5d43814b6b819de, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/f8730071c28544f4bf6c5db4264f6e97, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/45189ca8d83347d398a3c1b9bca862a1, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/49dd085f71854997b3b3d52fc25e6d13, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/cb54fdd5050c4042a518e874fe8923d7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/4fb0c02b9c274c9682b5fceb32cf2309, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/f450dd05d0614316a3be90569a3e552b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/1d15abbb719345139a949c401f8408bb, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/1258eb7e9a7e40948f23a98f81e2a60c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/c8c51da9608f418aa9af421b5847c4d2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/3abb7680a3fd410bb88283f119024e47, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/ee9239ab78c94540b0a56ac8a866a390, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/87c69c3b6488419e835248a270ce5143, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/39fd00d9cbca4380b4112e3e8cc9d0a3, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/b2609e221ded4004b46fd04c75ed8367, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/62e2918b1dde4c11ab1a4d8c1ba299a4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/7e3ee07f323d48b5b3aeed0fc5385374, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/9ed7481c481242a9826ef865bc5aadda, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/61912aeff1c7430a9f3965d1eae94ee3, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/9b2c04c2ef90484283ce9b1ffdfc95a0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/292b37f8a2624690a67250b7fc4ef6b6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/15d69b296e1f48ae97c2b4ffaa43086e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/db9d0c5c91db4e1aa0fd4e5c56b96e1f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/e150c0c912b14488a589cb6a6f429ddc, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/eed82c067e7a4cc3abbc95f7fd44eb2b] to archive 2024-11-15T22:11:57,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:11:57,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/baf192ea83744b11a5d43814b6b819de to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/baf192ea83744b11a5d43814b6b819de 2024-11-15T22:11:57,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/f8730071c28544f4bf6c5db4264f6e97 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/f8730071c28544f4bf6c5db4264f6e97 2024-11-15T22:11:57,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/45189ca8d83347d398a3c1b9bca862a1 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/45189ca8d83347d398a3c1b9bca862a1 2024-11-15T22:11:57,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/49dd085f71854997b3b3d52fc25e6d13 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/49dd085f71854997b3b3d52fc25e6d13 2024-11-15T22:11:57,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/cb54fdd5050c4042a518e874fe8923d7 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/cb54fdd5050c4042a518e874fe8923d7 2024-11-15T22:11:57,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/4fb0c02b9c274c9682b5fceb32cf2309 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/4fb0c02b9c274c9682b5fceb32cf2309 2024-11-15T22:11:57,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/f450dd05d0614316a3be90569a3e552b to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/f450dd05d0614316a3be90569a3e552b 2024-11-15T22:11:57,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/1d15abbb719345139a949c401f8408bb to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/1d15abbb719345139a949c401f8408bb 2024-11-15T22:11:57,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/1258eb7e9a7e40948f23a98f81e2a60c to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/1258eb7e9a7e40948f23a98f81e2a60c 2024-11-15T22:11:57,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/c8c51da9608f418aa9af421b5847c4d2 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/c8c51da9608f418aa9af421b5847c4d2 2024-11-15T22:11:57,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/3abb7680a3fd410bb88283f119024e47 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/3abb7680a3fd410bb88283f119024e47 2024-11-15T22:11:57,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/ee9239ab78c94540b0a56ac8a866a390 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/ee9239ab78c94540b0a56ac8a866a390 2024-11-15T22:11:57,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/87c69c3b6488419e835248a270ce5143 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/87c69c3b6488419e835248a270ce5143 2024-11-15T22:11:57,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/39fd00d9cbca4380b4112e3e8cc9d0a3 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/39fd00d9cbca4380b4112e3e8cc9d0a3 2024-11-15T22:11:57,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/b2609e221ded4004b46fd04c75ed8367 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/b2609e221ded4004b46fd04c75ed8367 2024-11-15T22:11:57,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/62e2918b1dde4c11ab1a4d8c1ba299a4 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/62e2918b1dde4c11ab1a4d8c1ba299a4 2024-11-15T22:11:57,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/7e3ee07f323d48b5b3aeed0fc5385374 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/7e3ee07f323d48b5b3aeed0fc5385374 2024-11-15T22:11:57,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/9ed7481c481242a9826ef865bc5aadda to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/9ed7481c481242a9826ef865bc5aadda 2024-11-15T22:11:57,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/61912aeff1c7430a9f3965d1eae94ee3 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/61912aeff1c7430a9f3965d1eae94ee3 2024-11-15T22:11:57,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/9b2c04c2ef90484283ce9b1ffdfc95a0 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/9b2c04c2ef90484283ce9b1ffdfc95a0 2024-11-15T22:11:57,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/292b37f8a2624690a67250b7fc4ef6b6 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/292b37f8a2624690a67250b7fc4ef6b6 2024-11-15T22:11:57,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/15d69b296e1f48ae97c2b4ffaa43086e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/15d69b296e1f48ae97c2b4ffaa43086e 2024-11-15T22:11:57,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/db9d0c5c91db4e1aa0fd4e5c56b96e1f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/db9d0c5c91db4e1aa0fd4e5c56b96e1f 2024-11-15T22:11:57,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/e150c0c912b14488a589cb6a6f429ddc to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/e150c0c912b14488a589cb6a6f429ddc 2024-11-15T22:11:57,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/eed82c067e7a4cc3abbc95f7fd44eb2b to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/eed82c067e7a4cc3abbc95f7fd44eb2b 2024-11-15T22:11:57,506 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/recovered.edits/378.seqid, newMaxSeqId=378, maxSeqId=1 2024-11-15T22:11:57,507 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. 2024-11-15T22:11:57,507 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for 3220684e1e7cfe8432a04d7ee5eae414: Waiting for close lock at 1731708716070Running coprocessor pre-close hooks at 1731708716070Disabling compacts and flushes for region at 1731708716070Disabling writes for close at 1731708716071 (+1 ms)Obtaining lock to block concurrent updates at 1731708716071Preparing flush snapshotting stores in 3220684e1e7cfe8432a04d7ee5eae414 at 1731708716071Finished memstore snapshotting TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731708716071Flushing stores of TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414. at 1731708716072 (+1 ms)Flushing 3220684e1e7cfe8432a04d7ee5eae414/A: creating writer at 1731708716072Flushing 3220684e1e7cfe8432a04d7ee5eae414/A: appending metadata at 1731708716075 (+3 ms)Flushing 3220684e1e7cfe8432a04d7ee5eae414/A: closing flushed file at 1731708716075Flushing 3220684e1e7cfe8432a04d7ee5eae414/B: creating writer at 1731708716486 (+411 ms)Flushing 3220684e1e7cfe8432a04d7ee5eae414/B: appending metadata at 1731708716489 (+3 ms)Flushing 3220684e1e7cfe8432a04d7ee5eae414/B: closing flushed file at 1731708716489Flushing 3220684e1e7cfe8432a04d7ee5eae414/C: creating writer at 1731708716900 (+411 ms)Flushing 3220684e1e7cfe8432a04d7ee5eae414/C: appending metadata at 1731708716902 (+2 ms)Flushing 3220684e1e7cfe8432a04d7ee5eae414/C: closing flushed file at 1731708716902Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@480a805c: reopening flushed file at 1731708717314 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f879c60: reopening flushed file at 1731708717321 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ad99f77: reopening flushed file at 1731708717330 (+9 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 3220684e1e7cfe8432a04d7ee5eae414 in 1267ms, sequenceid=375, compaction requested=false at 1731708717338 (+8 ms)Writing region close event to WAL at 1731708717499 (+161 ms)Running coprocessor post-close hooks at 1731708717506 (+7 ms)Closed at 1731708717507 (+1 ms) 2024-11-15T22:11:57,510 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed 3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:57,511 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=3220684e1e7cfe8432a04d7ee5eae414, regionState=CLOSED 2024-11-15T22:11:57,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 because future has completed 2024-11-15T22:11:57,518 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-15T22:11:57,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure 3220684e1e7cfe8432a04d7ee5eae414, server=ae3311ea5ccb,37917,1731708686516 in 1.6030 sec 2024-11-15T22:11:57,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-15T22:11:57,522 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3220684e1e7cfe8432a04d7ee5eae414, UNASSIGN in 1.6130 sec 2024-11-15T22:11:57,526 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-15T22:11:57,526 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6230 sec 2024-11-15T22:11:57,527 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708717527"}]},"ts":"1731708717527"} 2024-11-15T22:11:57,529 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T22:11:57,529 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T22:11:57,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6460 sec 2024-11-15T22:11:58,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T22:11:58,057 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:11:58,060 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T22:11:58,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:11:58,066 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:11:58,068 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:11:58,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T22:11:58,072 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:58,077 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A, FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B, FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C, FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/recovered.edits] 2024-11-15T22:11:58,080 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/034cc55db7604df3aa48409ca24d7f4a to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/034cc55db7604df3aa48409ca24d7f4a 2024-11-15T22:11:58,082 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/74b99484c0254d5f97b6d801e4d46d2a to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/A/74b99484c0254d5f97b6d801e4d46d2a 2024-11-15T22:11:58,085 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/1e5c92d2bd2d4fe2b0f8e3debb2774e3 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/1e5c92d2bd2d4fe2b0f8e3debb2774e3 2024-11-15T22:11:58,087 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/8a800bf056674b5d920a17d61ad35606 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/B/8a800bf056674b5d920a17d61ad35606 2024-11-15T22:11:58,089 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/98d305ed599940f98f1492ea5a0024a3 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/98d305ed599940f98f1492ea5a0024a3 2024-11-15T22:11:58,090 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/ed4979ebe0554e6aaa3472251f2ab0df to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/C/ed4979ebe0554e6aaa3472251f2ab0df 2024-11-15T22:11:58,094 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/recovered.edits/378.seqid to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414/recovered.edits/378.seqid 2024-11-15T22:11:58,095 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/3220684e1e7cfe8432a04d7ee5eae414 2024-11-15T22:11:58,095 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T22:11:58,101 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:11:58,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-15T22:11:58,112 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T22:11:58,147 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T22:11:58,149 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:11:58,149 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T22:11:58,150 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731708718150"}]},"ts":"9223372036854775807"} 2024-11-15T22:11:58,155 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T22:11:58,155 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3220684e1e7cfe8432a04d7ee5eae414, NAME => 'TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T22:11:58,155 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T22:11:58,156 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731708718156"}]},"ts":"9223372036854775807"} 2024-11-15T22:11:58,159 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T22:11:58,160 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:11:58,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 99 msec 2024-11-15T22:11:58,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T22:11:58,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T22:11:58,178 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:11:58,191 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=241 (was 211) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-361613644_22 at /127.0.0.1:36496 [Waiting for operation #233] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2050174287_22 at /127.0.0.1:54478 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-361613644_22 at /127.0.0.1:54498 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2050174287_22 at /127.0.0.1:54488 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-361613644_22 at /127.0.0.1:42558 [Waiting for operation #243] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;ae3311ea5ccb:37917-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=463 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=666 (was 535) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2724 (was 4099) 2024-11-15T22:11:58,202 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=241, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=666, ProcessCount=11, AvailableMemoryMB=2724 2024-11-15T22:11:58,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T22:11:58,205 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:11:58,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T22:11:58,207 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T22:11:58,208 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:11:58,208 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-15T22:11:58,209 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T22:11:58,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T22:11:58,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741918_1094 (size=959) 2024-11-15T22:11:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T22:11:58,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T22:11:58,619 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31 2024-11-15T22:11:58,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741919_1095 (size=53) 2024-11-15T22:11:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T22:11:59,032 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:11:59,032 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2a24e00e990dba09c516fd169bc5958d, disabling compactions & flushes 2024-11-15T22:11:59,032 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:11:59,032 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:11:59,032 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. after waiting 0 ms 2024-11-15T22:11:59,032 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:11:59,032 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:11:59,032 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2a24e00e990dba09c516fd169bc5958d: Waiting for close lock at 1731708719032Disabling compacts and flushes for region at 1731708719032Disabling writes for close at 1731708719032Writing region close event to WAL at 1731708719032Closed at 1731708719032 2024-11-15T22:11:59,034 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T22:11:59,034 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731708719034"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708719034"}]},"ts":"1731708719034"} 2024-11-15T22:11:59,036 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T22:11:59,038 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T22:11:59,038 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708719038"}]},"ts":"1731708719038"} 2024-11-15T22:11:59,040 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T22:11:59,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a24e00e990dba09c516fd169bc5958d, ASSIGN}] 2024-11-15T22:11:59,042 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a24e00e990dba09c516fd169bc5958d, ASSIGN 2024-11-15T22:11:59,043 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a24e00e990dba09c516fd169bc5958d, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,37917,1731708686516; forceNewPlan=false, retain=false 2024-11-15T22:11:59,193 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=2a24e00e990dba09c516fd169bc5958d, regionState=OPENING, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:59,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a24e00e990dba09c516fd169bc5958d, ASSIGN because future has completed 2024-11-15T22:11:59,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516}] 2024-11-15T22:11:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T22:11:59,352 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:11:59,352 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:11:59,353 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:11:59,353 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:11:59,353 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:11:59,353 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:11:59,354 INFO [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:11:59,357 INFO [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a24e00e990dba09c516fd169bc5958d columnFamilyName A 2024-11-15T22:11:59,357 DEBUG [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:11:59,357 INFO [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] regionserver.HStore(327): Store=2a24e00e990dba09c516fd169bc5958d/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:11:59,358 INFO [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:11:59,359 INFO [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a24e00e990dba09c516fd169bc5958d columnFamilyName B 2024-11-15T22:11:59,359 DEBUG [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:11:59,359 INFO [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] regionserver.HStore(327): Store=2a24e00e990dba09c516fd169bc5958d/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:11:59,359 INFO [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:11:59,361 INFO [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a24e00e990dba09c516fd169bc5958d columnFamilyName C 2024-11-15T22:11:59,361 DEBUG [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:11:59,361 INFO [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] regionserver.HStore(327): Store=2a24e00e990dba09c516fd169bc5958d/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:11:59,361 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:11:59,362 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:11:59,362 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:11:59,363 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:11:59,363 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:11:59,364 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:11:59,365 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:11:59,367 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:11:59,367 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 2a24e00e990dba09c516fd169bc5958d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58721236, jitterRate=-0.12498539686203003}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:11:59,367 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:11:59,368 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 2a24e00e990dba09c516fd169bc5958d: Running coprocessor pre-open hook at 1731708719353Writing region info on filesystem at 1731708719353Initializing all the Stores at 1731708719354 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708719354Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708719354Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708719354Cleaning up temporary data from old regions at 1731708719363 (+9 ms)Running coprocessor post-open hooks at 1731708719367 (+4 ms)Region opened successfully at 1731708719368 (+1 ms) 2024-11-15T22:11:59,369 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., pid=30, masterSystemTime=1731708719348 2024-11-15T22:11:59,371 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:11:59,371 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:11:59,372 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=2a24e00e990dba09c516fd169bc5958d, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:11:59,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 because future has completed 2024-11-15T22:11:59,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-15T22:11:59,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 in 178 msec 2024-11-15T22:11:59,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-15T22:11:59,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a24e00e990dba09c516fd169bc5958d, ASSIGN in 336 msec 2024-11-15T22:11:59,379 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T22:11:59,379 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708719379"}]},"ts":"1731708719379"} 2024-11-15T22:11:59,381 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T22:11:59,381 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T22:11:59,383 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-11-15T22:12:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T22:12:00,357 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:00,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57aa422e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:00,358 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:12:00,359 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:12:00,361 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:12:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:12:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:12:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cade026, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:12:00,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:12:00,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:00,363 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:12:00,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581f3f89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:00,365 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:12:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:12:00,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:12:00,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:12:00,369 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T22:12:00,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is ae3311ea5ccb,37303,1731708685620 2024-11-15T22:12:00,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3aa19d3a 2024-11-15T22:12:00,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T22:12:00,372 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T22:12:00,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T22:12:00,378 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:12:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T22:12:00,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741920_1096 (size=995) 2024-11-15T22:12:00,818 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-15T22:12:00,818 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-15T22:12:00,824 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T22:12:00,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a24e00e990dba09c516fd169bc5958d, REOPEN/MOVE}] 2024-11-15T22:12:00,836 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a24e00e990dba09c516fd169bc5958d, REOPEN/MOVE 2024-11-15T22:12:00,837 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=2a24e00e990dba09c516fd169bc5958d, regionState=CLOSING, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:00,839 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a24e00e990dba09c516fd169bc5958d, REOPEN/MOVE because future has completed 2024-11-15T22:12:00,844 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T22:12:00,844 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516}] 2024-11-15T22:12:00,999 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:00,999 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T22:12:00,999 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 2a24e00e990dba09c516fd169bc5958d, disabling compactions & flushes 2024-11-15T22:12:00,999 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:00,999 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:00,999 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. after waiting 0 ms 2024-11-15T22:12:00,999 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:01,005 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:12:01,006 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:01,006 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 2a24e00e990dba09c516fd169bc5958d: Waiting for close lock at 1731708720999Running coprocessor pre-close hooks at 1731708720999Disabling compacts and flushes for region at 1731708720999Disabling writes for close at 1731708720999Writing region close event to WAL at 1731708721000 (+1 ms)Running coprocessor post-close hooks at 1731708721005 (+5 ms)Closed at 1731708721006 (+1 ms) 2024-11-15T22:12:01,006 WARN [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: 2a24e00e990dba09c516fd169bc5958d to self. 2024-11-15T22:12:01,009 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:01,010 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=2a24e00e990dba09c516fd169bc5958d, regionState=CLOSED 2024-11-15T22:12:01,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 because future has completed 2024-11-15T22:12:01,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-15T22:12:01,017 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a24e00e990dba09c516fd169bc5958d, REOPEN/MOVE; state=CLOSED, location=ae3311ea5ccb,37917,1731708686516; forceNewPlan=false, retain=true 2024-11-15T22:12:01,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 in 169 msec 2024-11-15T22:12:01,168 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=2a24e00e990dba09c516fd169bc5958d, regionState=OPENING, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:01,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a24e00e990dba09c516fd169bc5958d, REOPEN/MOVE because future has completed 2024-11-15T22:12:01,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516}] 2024-11-15T22:12:01,328 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:01,329 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:12:01,329 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:01,329 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:12:01,329 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:01,329 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:01,333 INFO [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:01,340 INFO [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a24e00e990dba09c516fd169bc5958d columnFamilyName A 2024-11-15T22:12:01,353 DEBUG [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:01,356 INFO [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] regionserver.HStore(327): Store=2a24e00e990dba09c516fd169bc5958d/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:12:01,361 INFO [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:01,364 INFO [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a24e00e990dba09c516fd169bc5958d columnFamilyName B 2024-11-15T22:12:01,364 DEBUG [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:01,366 INFO [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] regionserver.HStore(327): Store=2a24e00e990dba09c516fd169bc5958d/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:12:01,366 INFO [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:01,367 INFO [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a24e00e990dba09c516fd169bc5958d columnFamilyName C 2024-11-15T22:12:01,367 DEBUG [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:01,368 INFO [StoreOpener-2a24e00e990dba09c516fd169bc5958d-1 {}] regionserver.HStore(327): Store=2a24e00e990dba09c516fd169bc5958d/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:12:01,368 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:01,369 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:01,370 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:01,372 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:01,372 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:01,372 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:12:01,374 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:01,375 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 2a24e00e990dba09c516fd169bc5958d; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60344016, jitterRate=-0.10080409049987793}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:12:01,375 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:01,377 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 2a24e00e990dba09c516fd169bc5958d: Running coprocessor pre-open hook at 1731708721330Writing region info on filesystem at 1731708721330Initializing all the Stores at 1731708721331 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708721331Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708721331Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708721331Cleaning up temporary data from old regions at 1731708721372 (+41 ms)Running coprocessor post-open hooks at 1731708721375 (+3 ms)Region opened successfully at 1731708721376 (+1 ms) 2024-11-15T22:12:01,378 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., pid=35, masterSystemTime=1731708721324 2024-11-15T22:12:01,380 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:01,380 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:01,381 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=2a24e00e990dba09c516fd169bc5958d, regionState=OPEN, openSeqNum=5, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:01,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 because future has completed 2024-11-15T22:12:01,387 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-15T22:12:01,387 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 in 213 msec 2024-11-15T22:12:01,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-15T22:12:01,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a24e00e990dba09c516fd169bc5958d, REOPEN/MOVE in 553 msec 2024-11-15T22:12:01,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-15T22:12:01,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 568 msec 2024-11-15T22:12:01,406 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0210 sec 2024-11-15T22:12:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-15T22:12:01,414 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:01,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13e89b68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,415 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:12:01,415 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:12:01,427 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:12:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:12:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:12:01,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cd7edbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:12:01,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:12:01,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:01,434 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:12:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@389942aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4039272, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,436 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:12:01,436 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:12:01,438 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:12:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:12:01,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:12:01,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3815d824, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:12:01,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:12:01,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:01,442 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:12:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48edf00c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46a8e9f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,445 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:12:01,446 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:12:01,450 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:12:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:12:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:12:01,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c42aae0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:12:01,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:12:01,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:01,452 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:12:01,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f7753d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a179463, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,454 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:12:01,455 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:12:01,456 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:12:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:12:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:12:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cfb0af6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:12:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:12:01,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:01,458 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:12:01,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c0d489c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d7d4682, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,459 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:12:01,459 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:12:01,460 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:12:01,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:12:01,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:12:01,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c01bd33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:12:01,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:12:01,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:01,462 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:12:01,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39607d73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38bb1855, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,464 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:12:01,464 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:12:01,470 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:12:01,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:12:01,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:12:01,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6042a0fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:12:01,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:12:01,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:01,476 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:12:01,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69a9b349, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,481 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b3a2f0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,482 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:12:01,482 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:12:01,483 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:12:01,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:12:01,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:12:01,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2194f95b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:12:01,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:12:01,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:01,490 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:12:01,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e7a4b3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,497 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5afa5df8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,497 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:12:01,497 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:12:01,498 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:12:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:12:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:12:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1f9e53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:12:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:12:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:01,510 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:12:01,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a52b838, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,514 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60e92ea3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,514 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:12:01,514 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:12:01,518 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:12:01,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:12:01,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:12:01,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b00f32c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:12:01,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:12:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:01,521 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:12:01,522 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3035364d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:01,530 DEBUG [Thread-517 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:12:01,530 DEBUG [Thread-534 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:12:01,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:12:01,531 DEBUG [Thread-521 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:12:01,532 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:12:01,534 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:12:01,536 DEBUG [Thread-523 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:12:01,536 DEBUG [Thread-521 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:12:01,537 DEBUG [Thread-521 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:12:01,539 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:12:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:12:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:12:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:12:01,541 DEBUG [Thread-528 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:12:01,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:12:01,542 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:12:01,543 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:12:01,544 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:12:01,545 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:12:01,546 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:12:01,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:12:01,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:12:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:12:01,549 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:12:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:12:01,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:12:01,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-15T22:12:01,560 DEBUG [Thread-532 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:12:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T22:12:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:12:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:12:01,563 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:12:01,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:12:01,566 DEBUG [Thread-519 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:12:01,568 DEBUG [Thread-525 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:12:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:12:01,568 DEBUG [Thread-530 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:12:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:12:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:12:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:12:01,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:12:01,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:12:01,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:12:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:12:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:12:01,577 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:12:01,579 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:12:01,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:12:01,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:12:01,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:12:01,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:12:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:01,597 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:01,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T22:12:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T22:12:01,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115860542042b8b4edbb65f75b09c7062da_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708721596/Put/seqid=0 2024-11-15T22:12:01,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708781701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:01,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:01,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:01,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708781699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:01,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:01,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708781710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:01,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:01,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:01,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:01,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:01,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708781712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:01,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:01,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:01,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:01,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:01,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741921_1097 (size=12154) 2024-11-15T22:12:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708781714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:01,732 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:01,742 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115860542042b8b4edbb65f75b09c7062da_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115860542042b8b4edbb65f75b09c7062da_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:01,746 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/0b2bf20d3f144011a2263a9cfb34fc22, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:01,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T22:12:01,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:01,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:01,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:01,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:01,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:01,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/0b2bf20d3f144011a2263a9cfb34fc22 is 175, key is test_row_0/A:col10/1731708721596/Put/seqid=0 2024-11-15T22:12:01,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741922_1098 (size=30955) 2024-11-15T22:12:01,801 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/0b2bf20d3f144011a2263a9cfb34fc22 2024-11-15T22:12:01,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708781817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:01,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:01,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:01,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:01,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708781826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:01,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708781837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:01,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708781837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:01,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708781838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:01,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:01,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:01,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:01,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:01,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:01,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:01,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/3617d1b73dae42fc9dccb219c0100b1e is 50, key is test_row_0/B:col10/1731708721596/Put/seqid=0 2024-11-15T22:12:01,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T22:12:01,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T22:12:01,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:01,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:01,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:01,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:01,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:01,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741923_1099 (size=12001) 2024-11-15T22:12:01,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/3617d1b73dae42fc9dccb219c0100b1e 2024-11-15T22:12:02,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708782036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:02,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/f9b316d5f7404284a543cd02319c8e72 is 50, key is test_row_0/C:col10/1731708721596/Put/seqid=0 2024-11-15T22:12:02,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:02,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708782057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:02,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708782067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708782068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708782068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:02,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T22:12:02,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:02,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:02,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:02,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:02,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:02,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741924_1100 (size=12001) 2024-11-15T22:12:02,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/f9b316d5f7404284a543cd02319c8e72 2024-11-15T22:12:02,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/0b2bf20d3f144011a2263a9cfb34fc22 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/0b2bf20d3f144011a2263a9cfb34fc22 2024-11-15T22:12:02,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/0b2bf20d3f144011a2263a9cfb34fc22, entries=150, sequenceid=17, filesize=30.2 K 2024-11-15T22:12:02,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/3617d1b73dae42fc9dccb219c0100b1e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/3617d1b73dae42fc9dccb219c0100b1e 2024-11-15T22:12:02,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/3617d1b73dae42fc9dccb219c0100b1e, entries=150, sequenceid=17, filesize=11.7 K 2024-11-15T22:12:02,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/f9b316d5f7404284a543cd02319c8e72 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/f9b316d5f7404284a543cd02319c8e72 2024-11-15T22:12:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T22:12:02,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/f9b316d5f7404284a543cd02319c8e72, entries=150, sequenceid=17, filesize=11.7 K 2024-11-15T22:12:02,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 2a24e00e990dba09c516fd169bc5958d in 624ms, sequenceid=17, compaction requested=false 2024-11-15T22:12:02,222 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T22:12:02,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:02,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T22:12:02,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:02,246 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T22:12:02,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111557e044d1456b487888120a412305810a_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708721689/Put/seqid=0 2024-11-15T22:12:02,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741925_1101 (size=12154) 2024-11-15T22:12:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:02,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:02,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:02,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:02,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708782404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708782402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708782407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:02,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:02,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:02,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708782419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:02,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708782419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:02,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:02,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:02,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:02,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708782527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:02,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:02,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708782528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:02,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:02,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708782530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:02,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:02,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:02,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:02,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708782530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:02,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:02,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:02,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708782529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:02,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T22:12:02,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:02,725 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111557e044d1456b487888120a412305810a_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111557e044d1456b487888120a412305810a_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:02,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/3de55b29846b4c90862b0f7f18593cca, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:02,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/3de55b29846b4c90862b0f7f18593cca is 175, key is test_row_0/A:col10/1731708721689/Put/seqid=0 2024-11-15T22:12:02,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708782748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:02,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:02,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708782748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:02,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:02,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:02,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708782748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:02,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:02,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708782749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:02,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:02,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:02,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708782749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:02,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741926_1102 (size=30955) 2024-11-15T22:12:03,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:03,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708783067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:03,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:03,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:03,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:03,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:03,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708783068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:03,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708783067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:03,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:03,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708783068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708783068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:03,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:03,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:03,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:03,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:03,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:03,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:03,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:03,098 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T22:12:03,177 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/3de55b29846b4c90862b0f7f18593cca 2024-11-15T22:12:03,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/4163c40179b1444e82e7ccc856c2e08f is 50, key is test_row_0/B:col10/1731708721689/Put/seqid=0 2024-11-15T22:12:03,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741927_1103 (size=12001) 2024-11-15T22:12:03,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:03,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708783577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:03,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:03,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:03,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:03,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708783577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:03,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:03,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:03,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:03,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708783587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:03,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708783588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:03,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708783588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:03,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:03,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:03,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:03,621 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/4163c40179b1444e82e7ccc856c2e08f 2024-11-15T22:12:03,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/8a35bc740b5a47dc8bce92ccad3d63a5 is 50, key is test_row_0/C:col10/1731708721689/Put/seqid=0 2024-11-15T22:12:03,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741928_1104 (size=12001) 2024-11-15T22:12:03,698 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/8a35bc740b5a47dc8bce92ccad3d63a5 2024-11-15T22:12:03,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/3de55b29846b4c90862b0f7f18593cca as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/3de55b29846b4c90862b0f7f18593cca 2024-11-15T22:12:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T22:12:03,720 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/3de55b29846b4c90862b0f7f18593cca, entries=150, sequenceid=40, filesize=30.2 K 2024-11-15T22:12:03,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/4163c40179b1444e82e7ccc856c2e08f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/4163c40179b1444e82e7ccc856c2e08f 2024-11-15T22:12:03,728 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/4163c40179b1444e82e7ccc856c2e08f, entries=150, sequenceid=40, filesize=11.7 K 2024-11-15T22:12:03,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/8a35bc740b5a47dc8bce92ccad3d63a5 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/8a35bc740b5a47dc8bce92ccad3d63a5 2024-11-15T22:12:03,737 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/8a35bc740b5a47dc8bce92ccad3d63a5, entries=150, sequenceid=40, filesize=11.7 K 2024-11-15T22:12:03,742 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 2a24e00e990dba09c516fd169bc5958d in 1492ms, sequenceid=40, compaction requested=false 2024-11-15T22:12:03,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:03,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:03,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-15T22:12:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-15T22:12:03,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-15T22:12:03,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1670 sec 2024-11-15T22:12:03,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.2040 sec 2024-11-15T22:12:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:04,595 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:04,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T22:12:04,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b1409c8daafe4c2fb4d2fa5aad60b681_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708722414/Put/seqid=0 2024-11-15T22:12:04,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741929_1105 (size=14594) 2024-11-15T22:12:04,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:04,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708784652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:04,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:04,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:04,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708784654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:04,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:04,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708784661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:04,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708784662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:04,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708784664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:04,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:04,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708784766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:04,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:04,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:04,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708784777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:04,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:04,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:04,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708784777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:04,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:04,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708784778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:04,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708784778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:04,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:04,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708784977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:04,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:04,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708784987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:04,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708784987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:04,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:04,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708784987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:04,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708784987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:04,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:05,065 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:05,073 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b1409c8daafe4c2fb4d2fa5aad60b681_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b1409c8daafe4c2fb4d2fa5aad60b681_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:05,074 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/77f6eef5fd6b483a8bb09b69a4d1b3fb, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:05,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/77f6eef5fd6b483a8bb09b69a4d1b3fb is 175, key is test_row_0/A:col10/1731708722414/Put/seqid=0 2024-11-15T22:12:05,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741930_1106 (size=39549) 2024-11-15T22:12:05,124 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/77f6eef5fd6b483a8bb09b69a4d1b3fb 2024-11-15T22:12:05,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/291f0eae3c494da0ac8c32cc04e7058f is 50, key is test_row_0/B:col10/1731708722414/Put/seqid=0 2024-11-15T22:12:05,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741931_1107 (size=12001) 2024-11-15T22:12:05,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/291f0eae3c494da0ac8c32cc04e7058f 2024-11-15T22:12:05,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/259b92e04b1e4bb7aef3f51ef2dc5464 is 50, key is test_row_0/C:col10/1731708722414/Put/seqid=0 2024-11-15T22:12:05,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741932_1108 (size=12001) 2024-11-15T22:12:05,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708785297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:05,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:05,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708785297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:05,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708785297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:05,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708785297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708785297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:05,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/259b92e04b1e4bb7aef3f51ef2dc5464 2024-11-15T22:12:05,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/77f6eef5fd6b483a8bb09b69a4d1b3fb as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/77f6eef5fd6b483a8bb09b69a4d1b3fb 2024-11-15T22:12:05,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/77f6eef5fd6b483a8bb09b69a4d1b3fb, entries=200, sequenceid=55, filesize=38.6 K 2024-11-15T22:12:05,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/291f0eae3c494da0ac8c32cc04e7058f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/291f0eae3c494da0ac8c32cc04e7058f 2024-11-15T22:12:05,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/291f0eae3c494da0ac8c32cc04e7058f, entries=150, sequenceid=55, filesize=11.7 K 2024-11-15T22:12:05,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/259b92e04b1e4bb7aef3f51ef2dc5464 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/259b92e04b1e4bb7aef3f51ef2dc5464 2024-11-15T22:12:05,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/259b92e04b1e4bb7aef3f51ef2dc5464, entries=150, sequenceid=55, filesize=11.7 K 2024-11-15T22:12:05,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 2a24e00e990dba09c516fd169bc5958d in 1105ms, sequenceid=55, compaction requested=true 2024-11-15T22:12:05,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:05,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:12:05,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:05,701 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:05,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:12:05,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:05,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:12:05,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:12:05,701 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:05,710 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:05,710 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/A is initiating minor compaction (all files) 2024-11-15T22:12:05,710 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/A in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:05,710 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/0b2bf20d3f144011a2263a9cfb34fc22, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/3de55b29846b4c90862b0f7f18593cca, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/77f6eef5fd6b483a8bb09b69a4d1b3fb] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=99.1 K 2024-11-15T22:12:05,710 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:05,711 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/0b2bf20d3f144011a2263a9cfb34fc22, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/3de55b29846b4c90862b0f7f18593cca, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/77f6eef5fd6b483a8bb09b69a4d1b3fb] 2024-11-15T22:12:05,711 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:05,711 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/B is initiating minor compaction (all files) 2024-11-15T22:12:05,711 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/B in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:05,711 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/3617d1b73dae42fc9dccb219c0100b1e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/4163c40179b1444e82e7ccc856c2e08f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/291f0eae3c494da0ac8c32cc04e7058f] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=35.2 K 2024-11-15T22:12:05,712 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b2bf20d3f144011a2263a9cfb34fc22, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731708721564 2024-11-15T22:12:05,716 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 3617d1b73dae42fc9dccb219c0100b1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731708721564 2024-11-15T22:12:05,717 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 4163c40179b1444e82e7ccc856c2e08f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731708721689 2024-11-15T22:12:05,717 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3de55b29846b4c90862b0f7f18593cca, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731708721689 2024-11-15T22:12:05,718 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77f6eef5fd6b483a8bb09b69a4d1b3fb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731708722386 2024-11-15T22:12:05,718 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 291f0eae3c494da0ac8c32cc04e7058f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731708722411 2024-11-15T22:12:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T22:12:05,740 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:05,746 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:12:05,746 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-15T22:12:05,750 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:12:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T22:12:05,751 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:12:05,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:12:05,752 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#B#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:05,752 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/b18568ebfc77476e95028bbb48bfda8d is 50, key is test_row_0/B:col10/1731708722414/Put/seqid=0 2024-11-15T22:12:05,758 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115cd45f047cc96485da46cd4f87d325896_2a24e00e990dba09c516fd169bc5958d store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:05,765 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115cd45f047cc96485da46cd4f87d325896_2a24e00e990dba09c516fd169bc5958d, store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:05,765 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cd45f047cc96485da46cd4f87d325896_2a24e00e990dba09c516fd169bc5958d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:05,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741933_1109 (size=12104) 2024-11-15T22:12:05,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741934_1110 (size=4469) 2024-11-15T22:12:05,816 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#A#compaction#90 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:05,817 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/aa9cd11997d0410f8aefd7bbac219861 is 175, key is test_row_0/A:col10/1731708722414/Put/seqid=0 2024-11-15T22:12:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:05,829 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:05,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T22:12:05,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T22:12:05,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115db8d182fde6e46f8ad680cf50bf551dd_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708725827/Put/seqid=0 2024-11-15T22:12:05,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741935_1111 (size=31058) 2024-11-15T22:12:05,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708785856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:05,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:05,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708785857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:05,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708785866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:05,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708785866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:05,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708785869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:05,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:05,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,883 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:05,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:05,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741936_1112 (size=12154) 2024-11-15T22:12:05,903 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:05,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T22:12:05,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:05,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:05,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:05,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:05,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:05,922 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115db8d182fde6e46f8ad680cf50bf551dd_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115db8d182fde6e46f8ad680cf50bf551dd_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:05,924 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/3bd5e16bf7f34abeb5f537d4bab391de, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:05,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/3bd5e16bf7f34abeb5f537d4bab391de is 175, key is test_row_0/A:col10/1731708725827/Put/seqid=0 2024-11-15T22:12:05,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741937_1113 (size=30955) 2024-11-15T22:12:05,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708785977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:05,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708785986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:05,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708785986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:05,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708785987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:05,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708785987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:05,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:05,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:06,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T22:12:06,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:06,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:06,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:06,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:06,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T22:12:06,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708786186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:06,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708786196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:06,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708786197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:06,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708786197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:06,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708786197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:06,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:06,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:06,213 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/b18568ebfc77476e95028bbb48bfda8d as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/b18568ebfc77476e95028bbb48bfda8d 2024-11-15T22:12:06,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T22:12:06,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:06,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:06,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:06,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:06,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:06,223 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/B of 2a24e00e990dba09c516fd169bc5958d into b18568ebfc77476e95028bbb48bfda8d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:06,224 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:06,224 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/B, priority=13, startTime=1731708725701; duration=0sec 2024-11-15T22:12:06,224 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:06,224 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:B 2024-11-15T22:12:06,224 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:06,226 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:06,226 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/C is initiating minor compaction (all files) 2024-11-15T22:12:06,226 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/C in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:06,226 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/f9b316d5f7404284a543cd02319c8e72, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/8a35bc740b5a47dc8bce92ccad3d63a5, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/259b92e04b1e4bb7aef3f51ef2dc5464] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=35.2 K 2024-11-15T22:12:06,227 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting f9b316d5f7404284a543cd02319c8e72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731708721564 2024-11-15T22:12:06,227 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a35bc740b5a47dc8bce92ccad3d63a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731708721689 2024-11-15T22:12:06,228 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 259b92e04b1e4bb7aef3f51ef2dc5464, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731708722411 2024-11-15T22:12:06,245 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#C#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:06,246 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/9c5a160417b349d68c710e54619dbfbb is 50, key is test_row_0/C:col10/1731708722414/Put/seqid=0 2024-11-15T22:12:06,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741938_1114 (size=12104) 2024-11-15T22:12:06,277 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/9c5a160417b349d68c710e54619dbfbb as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/9c5a160417b349d68c710e54619dbfbb 2024-11-15T22:12:06,281 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/aa9cd11997d0410f8aefd7bbac219861 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/aa9cd11997d0410f8aefd7bbac219861 2024-11-15T22:12:06,286 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/C of 2a24e00e990dba09c516fd169bc5958d into 9c5a160417b349d68c710e54619dbfbb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:06,287 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:06,287 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/C, priority=13, startTime=1731708725701; duration=0sec 2024-11-15T22:12:06,287 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:06,287 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:C 2024-11-15T22:12:06,298 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/A of 2a24e00e990dba09c516fd169bc5958d into aa9cd11997d0410f8aefd7bbac219861(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:06,298 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:06,298 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/A, priority=13, startTime=1731708725700; duration=0sec 2024-11-15T22:12:06,298 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:06,298 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:A 2024-11-15T22:12:06,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T22:12:06,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:06,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:06,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:06,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:06,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T22:12:06,382 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/3bd5e16bf7f34abeb5f537d4bab391de 2024-11-15T22:12:06,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/8d65933978b7460d97a379447b1a4e84 is 50, key is test_row_0/B:col10/1731708725827/Put/seqid=0 2024-11-15T22:12:06,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708786498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:06,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:06,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:06,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:06,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:06,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708786507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:06,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708786507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:06,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:06,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:06,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:06,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708786508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:06,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:06,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:06,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:06,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:06,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741939_1115 (size=12001) 2024-11-15T22:12:06,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708786517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:06,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:06,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:06,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:06,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T22:12:06,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:06,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:06,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:06,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:06,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:06,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:06,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T22:12:06,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:06,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:06,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:06,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:06,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:06,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T22:12:06,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:06,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:06,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:06,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:06,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:06,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T22:12:06,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/8d65933978b7460d97a379447b1a4e84 2024-11-15T22:12:06,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/a15896c2b2514407a8e837a65ac2b4bc is 50, key is test_row_0/C:col10/1731708725827/Put/seqid=0 2024-11-15T22:12:07,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T22:12:07,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:07,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:07,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:07,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:07,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:07,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:07,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708787007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:07,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:07,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:07,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:07,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:07,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708787024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:07,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708787026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:07,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:07,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:07,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:07,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:07,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:07,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:07,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741940_1116 (size=12001) 2024-11-15T22:12:07,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:07,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708787026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:07,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:07,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:07,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:07,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708787036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:07,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:07,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:07,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:07,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T22:12:07,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:07,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:07,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:07,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:07,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:07,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T22:12:07,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:07,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:07,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:07,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:07,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:07,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/a15896c2b2514407a8e837a65ac2b4bc 2024-11-15T22:12:07,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/3bd5e16bf7f34abeb5f537d4bab391de as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/3bd5e16bf7f34abeb5f537d4bab391de 2024-11-15T22:12:07,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/3bd5e16bf7f34abeb5f537d4bab391de, entries=150, sequenceid=80, filesize=30.2 K 2024-11-15T22:12:07,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/8d65933978b7460d97a379447b1a4e84 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/8d65933978b7460d97a379447b1a4e84 2024-11-15T22:12:07,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/8d65933978b7460d97a379447b1a4e84, entries=150, sequenceid=80, filesize=11.7 K 2024-11-15T22:12:07,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/a15896c2b2514407a8e837a65ac2b4bc as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/a15896c2b2514407a8e837a65ac2b4bc 2024-11-15T22:12:07,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T22:12:07,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:07,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:07,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:07,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:07,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:07,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/a15896c2b2514407a8e837a65ac2b4bc, entries=150, sequenceid=80, filesize=11.7 K 2024-11-15T22:12:07,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 2a24e00e990dba09c516fd169bc5958d in 1651ms, sequenceid=80, compaction requested=false 2024-11-15T22:12:07,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:07,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T22:12:07,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:07,630 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T22:12:07,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f072b63155974ea099a6bc377c152192_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708725853/Put/seqid=0 2024-11-15T22:12:07,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741941_1117 (size=12154) 2024-11-15T22:12:07,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T22:12:08,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:08,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:08,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708788083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:08,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:08,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:08,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708788086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:08,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:08,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708788088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:08,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:08,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:08,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708788088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:08,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:08,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708788098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:08,108 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f072b63155974ea099a6bc377c152192_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f072b63155974ea099a6bc377c152192_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:08,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/9ca4b834879e4cd1bf2c678ad2c63217, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:08,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/9ca4b834879e4cd1bf2c678ad2c63217 is 175, key is test_row_0/A:col10/1731708725853/Put/seqid=0 2024-11-15T22:12:08,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741942_1118 (size=30955) 2024-11-15T22:12:08,157 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/9ca4b834879e4cd1bf2c678ad2c63217 2024-11-15T22:12:08,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/d29b182c11af4fb5ad4ff20efbeec23f is 50, key is test_row_0/B:col10/1731708725853/Put/seqid=0 2024-11-15T22:12:08,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741943_1119 (size=12001) 2024-11-15T22:12:08,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708788197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:08,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:08,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:08,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708788207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:08,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:08,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:08,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708788207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:08,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:08,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708788208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:08,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:08,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708788208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:08,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:08,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:08,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:08,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708788409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:08,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:08,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708788416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:08,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:08,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708788416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:08,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:08,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:08,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708788417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:08,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708788418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:08,609 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/d29b182c11af4fb5ad4ff20efbeec23f 2024-11-15T22:12:08,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/121f286c4b0d4dd7ae5edb9b11869445 is 50, key is test_row_0/C:col10/1731708725853/Put/seqid=0 2024-11-15T22:12:08,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741944_1120 (size=12001) 2024-11-15T22:12:08,690 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/121f286c4b0d4dd7ae5edb9b11869445 2024-11-15T22:12:08,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/9ca4b834879e4cd1bf2c678ad2c63217 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/9ca4b834879e4cd1bf2c678ad2c63217 2024-11-15T22:12:08,715 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/9ca4b834879e4cd1bf2c678ad2c63217, entries=150, sequenceid=95, filesize=30.2 K 2024-11-15T22:12:08,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/d29b182c11af4fb5ad4ff20efbeec23f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/d29b182c11af4fb5ad4ff20efbeec23f 2024-11-15T22:12:08,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:08,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:08,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708788733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:08,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708788733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:08,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:08,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708788733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:08,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:08,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708788736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:08,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:08,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:08,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:08,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708788737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:08,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:08,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:08,748 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/d29b182c11af4fb5ad4ff20efbeec23f, entries=150, sequenceid=95, filesize=11.7 K 2024-11-15T22:12:08,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/121f286c4b0d4dd7ae5edb9b11869445 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/121f286c4b0d4dd7ae5edb9b11869445 2024-11-15T22:12:08,756 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/121f286c4b0d4dd7ae5edb9b11869445, entries=150, sequenceid=95, filesize=11.7 K 2024-11-15T22:12:08,758 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for 2a24e00e990dba09c516fd169bc5958d in 1128ms, sequenceid=95, compaction requested=true 2024-11-15T22:12:08,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:08,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:08,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-15T22:12:08,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-15T22:12:08,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-15T22:12:08,763 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0090 sec 2024-11-15T22:12:08,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 3.0230 sec 2024-11-15T22:12:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:09,249 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:09,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-15T22:12:09,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159fa45affa68f4c849339e1a74e364d34_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708729249/Put/seqid=0 2024-11-15T22:12:09,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708789257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:09,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:09,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708789258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:09,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708789264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:09,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741945_1121 (size=12154) 2024-11-15T22:12:09,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708789266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:09,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708789267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:09,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:09,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:09,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708789376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:09,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708789377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:09,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708789377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:09,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708789377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:09,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708789377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:09,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:09,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:09,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:09,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708789587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:09,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708789587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:09,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:09,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708789587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708789587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:09,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708789587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:09,672 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:09,677 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159fa45affa68f4c849339e1a74e364d34_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159fa45affa68f4c849339e1a74e364d34_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:09,678 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/670e59c3dbe74a04af50d7521e4e0a47, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:09,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/670e59c3dbe74a04af50d7521e4e0a47 is 175, key is test_row_0/A:col10/1731708729249/Put/seqid=0 2024-11-15T22:12:09,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741946_1122 (size=30955) 2024-11-15T22:12:09,756 INFO [master/ae3311ea5ccb:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-15T22:12:09,756 INFO [master/ae3311ea5ccb:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-15T22:12:09,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708789896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708789896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:09,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:09,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708789896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:09,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708789897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:09,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T22:12:09,907 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:09,908 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:12:09,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-15T22:12:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T22:12:09,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708789907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:09,911 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:12:09,912 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:12:09,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:12:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T22:12:10,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T22:12:10,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:10,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:10,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:10,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:10,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:10,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:10,087 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=123, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/670e59c3dbe74a04af50d7521e4e0a47 2024-11-15T22:12:10,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/76760b50a9ae464b96b035973c0c458a is 50, key is test_row_0/B:col10/1731708729249/Put/seqid=0 2024-11-15T22:12:10,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741947_1123 (size=12001) 2024-11-15T22:12:10,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/76760b50a9ae464b96b035973c0c458a 2024-11-15T22:12:10,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/32af93aa691645f6861e704b7c446c44 is 50, key is test_row_0/C:col10/1731708729249/Put/seqid=0 2024-11-15T22:12:10,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741948_1124 (size=12001) 2024-11-15T22:12:10,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/32af93aa691645f6861e704b7c446c44 2024-11-15T22:12:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T22:12:10,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T22:12:10,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:10,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:10,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:10,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:10,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:10,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:10,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/670e59c3dbe74a04af50d7521e4e0a47 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/670e59c3dbe74a04af50d7521e4e0a47 2024-11-15T22:12:10,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/670e59c3dbe74a04af50d7521e4e0a47, entries=150, sequenceid=123, filesize=30.2 K 2024-11-15T22:12:10,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/76760b50a9ae464b96b035973c0c458a as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/76760b50a9ae464b96b035973c0c458a 2024-11-15T22:12:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/76760b50a9ae464b96b035973c0c458a, entries=150, sequenceid=123, filesize=11.7 K 2024-11-15T22:12:10,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/32af93aa691645f6861e704b7c446c44 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/32af93aa691645f6861e704b7c446c44 2024-11-15T22:12:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/32af93aa691645f6861e704b7c446c44, entries=150, sequenceid=123, filesize=11.7 K 2024-11-15T22:12:10,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for 2a24e00e990dba09c516fd169bc5958d in 1027ms, sequenceid=123, compaction requested=true 2024-11-15T22:12:10,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:10,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:12:10,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:10,277 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:12:10,277 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:12:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:12:10,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:10,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:12:10,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,281 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:12:10,281 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/B is initiating minor compaction (all files) 2024-11-15T22:12:10,281 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/B in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:10,281 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/b18568ebfc77476e95028bbb48bfda8d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/8d65933978b7460d97a379447b1a4e84, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/d29b182c11af4fb5ad4ff20efbeec23f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/76760b50a9ae464b96b035973c0c458a] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=47.0 K 2024-11-15T22:12:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,281 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 123923 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:12:10,281 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/A is initiating minor compaction (all files) 2024-11-15T22:12:10,281 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/A in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:10,281 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/aa9cd11997d0410f8aefd7bbac219861, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/3bd5e16bf7f34abeb5f537d4bab391de, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/9ca4b834879e4cd1bf2c678ad2c63217, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/670e59c3dbe74a04af50d7521e4e0a47] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=121.0 K 2024-11-15T22:12:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,281 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:10,282 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/aa9cd11997d0410f8aefd7bbac219861, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/3bd5e16bf7f34abeb5f537d4bab391de, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/9ca4b834879e4cd1bf2c678ad2c63217, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/670e59c3dbe74a04af50d7521e4e0a47] 2024-11-15T22:12:10,282 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting b18568ebfc77476e95028bbb48bfda8d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731708722411 2024-11-15T22:12:10,282 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa9cd11997d0410f8aefd7bbac219861, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731708722411 2024-11-15T22:12:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,283 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d65933978b7460d97a379447b1a4e84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731708724650 2024-11-15T22:12:10,283 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bd5e16bf7f34abeb5f537d4bab391de, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731708724650 2024-11-15T22:12:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,284 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting d29b182c11af4fb5ad4ff20efbeec23f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731708725847 2024-11-15T22:12:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,284 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ca4b834879e4cd1bf2c678ad2c63217, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731708725847 2024-11-15T22:12:10,285 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 76760b50a9ae464b96b035973c0c458a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731708728081 2024-11-15T22:12:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,285 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 670e59c3dbe74a04af50d7521e4e0a47, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731708728081 2024-11-15T22:12:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,314 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#B#compaction#102 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:10,315 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/c1d0ec4ac1f848d38a3eabe41032ea96 is 50, key is test_row_0/B:col10/1731708729249/Put/seqid=0 2024-11-15T22:12:10,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,322 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:10,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,354 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115f829d111a0b74bce991b214f99bc6024_2a24e00e990dba09c516fd169bc5958d store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:10,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,364 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115f829d111a0b74bce991b214f99bc6024_2a24e00e990dba09c516fd169bc5958d, store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:10,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,364 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f829d111a0b74bce991b214f99bc6024_2a24e00e990dba09c516fd169bc5958d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741949_1125 (size=12241) 2024-11-15T22:12:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,377 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/c1d0ec4ac1f848d38a3eabe41032ea96 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/c1d0ec4ac1f848d38a3eabe41032ea96 2024-11-15T22:12:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T22:12:10,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:10,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,380 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-15T22:12:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,386 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/B of 2a24e00e990dba09c516fd169bc5958d into c1d0ec4ac1f848d38a3eabe41032ea96(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:10,386 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:10,386 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/B, priority=12, startTime=1731708730277; duration=0sec 2024-11-15T22:12:10,386 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:10,386 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:B 2024-11-15T22:12:10,386 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:12:10,387 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:12:10,388 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/C is initiating minor compaction (all files) 2024-11-15T22:12:10,388 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/C in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:10,388 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/9c5a160417b349d68c710e54619dbfbb, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/a15896c2b2514407a8e837a65ac2b4bc, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/121f286c4b0d4dd7ae5edb9b11869445, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/32af93aa691645f6861e704b7c446c44] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=47.0 K 2024-11-15T22:12:10,389 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c5a160417b349d68c710e54619dbfbb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731708722411 2024-11-15T22:12:10,390 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting a15896c2b2514407a8e837a65ac2b4bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731708724650 2024-11-15T22:12:10,391 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 121f286c4b0d4dd7ae5edb9b11869445, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731708725847 2024-11-15T22:12:10,392 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 32af93aa691645f6861e704b7c446c44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731708728081 2024-11-15T22:12:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741950_1126 (size=4469) 2024-11-15T22:12:10,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,409 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#A#compaction#103 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,411 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/14e12ed7675d460ab15ae5aee011892b is 175, key is test_row_0/A:col10/1731708729249/Put/seqid=0 2024-11-15T22:12:10,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:10,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:10,437 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#C#compaction#104 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,438 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/5cdc0d4bbfc143eda93acea988ceeafd is 50, key is test_row_0/C:col10/1731708729249/Put/seqid=0 2024-11-15T22:12:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111552f7877d5a664766a8adf050c4713a95_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708729262/Put/seqid=0 2024-11-15T22:12:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741951_1127 (size=31195) 2024-11-15T22:12:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741952_1128 (size=12241) 2024-11-15T22:12:10,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,472 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/14e12ed7675d460ab15ae5aee011892b as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/14e12ed7675d460ab15ae5aee011892b 2024-11-15T22:12:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,479 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/5cdc0d4bbfc143eda93acea988ceeafd as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/5cdc0d4bbfc143eda93acea988ceeafd 2024-11-15T22:12:10,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,483 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/A of 2a24e00e990dba09c516fd169bc5958d into 14e12ed7675d460ab15ae5aee011892b(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:10,483 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:10,484 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/A, priority=12, startTime=1731708730277; duration=0sec 2024-11-15T22:12:10,484 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:10,484 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:A 2024-11-15T22:12:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,486 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/C of 2a24e00e990dba09c516fd169bc5958d into 5cdc0d4bbfc143eda93acea988ceeafd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:10,486 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:10,486 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/C, priority=12, startTime=1731708730277; duration=0sec 2024-11-15T22:12:10,486 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:10,486 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:C 2024-11-15T22:12:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741953_1129 (size=9814) 2024-11-15T22:12:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,524 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111552f7877d5a664766a8adf050c4713a95_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111552f7877d5a664766a8adf050c4713a95_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/afe19724cccd4db99c06ac0e39c36eab, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:10,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/afe19724cccd4db99c06ac0e39c36eab is 175, key is test_row_0/A:col10/1731708729262/Put/seqid=0 2024-11-15T22:12:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T22:12:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708790529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708790530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:10,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708790533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708790533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:10,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:10,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708790535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:10,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:10,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741954_1130 (size=22461) 2024-11-15T22:12:10,557 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/afe19724cccd4db99c06ac0e39c36eab 2024-11-15T22:12:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/6ef2612327f1447fa33035bcac432061 is 50, key is test_row_0/B:col10/1731708729262/Put/seqid=0 2024-11-15T22:12:10,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741955_1131 (size=9757) 2024-11-15T22:12:10,627 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/6ef2612327f1447fa33035bcac432061 2024-11-15T22:12:10,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/bc510524fc1e4babb15fa107eb22ffd6 is 50, key is test_row_0/C:col10/1731708729262/Put/seqid=0 2024-11-15T22:12:10,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741956_1132 (size=9757) 2024-11-15T22:12:10,645 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/bc510524fc1e4babb15fa107eb22ffd6 2024-11-15T22:12:10,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708790647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:10,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:10,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/afe19724cccd4db99c06ac0e39c36eab as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/afe19724cccd4db99c06ac0e39c36eab 2024-11-15T22:12:10,658 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/afe19724cccd4db99c06ac0e39c36eab, entries=100, sequenceid=132, filesize=21.9 K 2024-11-15T22:12:10,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/6ef2612327f1447fa33035bcac432061 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/6ef2612327f1447fa33035bcac432061 2024-11-15T22:12:10,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708790658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:10,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:10,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708790658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:10,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708790658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:10,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708790659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:10,667 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/6ef2612327f1447fa33035bcac432061, entries=100, sequenceid=132, filesize=9.5 K 2024-11-15T22:12:10,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/bc510524fc1e4babb15fa107eb22ffd6 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/bc510524fc1e4babb15fa107eb22ffd6 2024-11-15T22:12:10,678 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/bc510524fc1e4babb15fa107eb22ffd6, entries=100, sequenceid=132, filesize=9.5 K 2024-11-15T22:12:10,684 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 2a24e00e990dba09c516fd169bc5958d in 304ms, sequenceid=132, compaction requested=false 2024-11-15T22:12:10,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:10,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:10,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-15T22:12:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-15T22:12:10,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-15T22:12:10,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 775 msec 2024-11-15T22:12:10,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 782 msec 2024-11-15T22:12:10,863 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:10,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-15T22:12:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:10,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e0d21cc6fb2347f28777b848649797ed_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708730531/Put/seqid=0 2024-11-15T22:12:10,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741957_1133 (size=14794) 2024-11-15T22:12:10,909 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:10,921 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e0d21cc6fb2347f28777b848649797ed_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e0d21cc6fb2347f28777b848649797ed_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:10,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:10,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708790884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:10,927 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/980ad2107b5442c8850a5571dd7e6111, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:10,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/980ad2107b5442c8850a5571dd7e6111 is 175, key is test_row_0/A:col10/1731708730531/Put/seqid=0 2024-11-15T22:12:10,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:10,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708790924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:10,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:10,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708790927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:10,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708790928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:10,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708790928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:10,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741958_1134 (size=39749) 2024-11-15T22:12:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:10,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:10,935 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=163, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/980ad2107b5442c8850a5571dd7e6111 2024-11-15T22:12:10,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/31d9f0088f574d69b500094bf7146f68 is 50, key is test_row_0/B:col10/1731708730531/Put/seqid=0 2024-11-15T22:12:10,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741959_1135 (size=12151) 2024-11-15T22:12:10,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/31d9f0088f574d69b500094bf7146f68 2024-11-15T22:12:10,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/4e6ab6ebf86a4f19b98ec85cc877cbf9 is 50, key is test_row_0/C:col10/1731708730531/Put/seqid=0 2024-11-15T22:12:11,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741960_1136 (size=12151) 2024-11-15T22:12:11,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/4e6ab6ebf86a4f19b98ec85cc877cbf9 2024-11-15T22:12:11,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/980ad2107b5442c8850a5571dd7e6111 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/980ad2107b5442c8850a5571dd7e6111 2024-11-15T22:12:11,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/980ad2107b5442c8850a5571dd7e6111, entries=200, sequenceid=163, filesize=38.8 K 2024-11-15T22:12:11,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/31d9f0088f574d69b500094bf7146f68 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/31d9f0088f574d69b500094bf7146f68 2024-11-15T22:12:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/31d9f0088f574d69b500094bf7146f68, entries=150, sequenceid=163, filesize=11.9 K 2024-11-15T22:12:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/4e6ab6ebf86a4f19b98ec85cc877cbf9 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/4e6ab6ebf86a4f19b98ec85cc877cbf9 2024-11-15T22:12:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/4e6ab6ebf86a4f19b98ec85cc877cbf9, entries=150, sequenceid=163, filesize=11.9 K 2024-11-15T22:12:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for 2a24e00e990dba09c516fd169bc5958d in 182ms, sequenceid=163, compaction requested=true 2024-11-15T22:12:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:11,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:12:11,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:11,045 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:11,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:12:11,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:11,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:12:11,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:12:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,045 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T22:12:11,046 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,047 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:12:11,047 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93405 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:11,047 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/A is initiating minor compaction (all files) 2024-11-15T22:12:11,047 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/A in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:11,048 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:11,048 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/B is initiating minor compaction (all files) 2024-11-15T22:12:11,048 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/B in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,048 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/c1d0ec4ac1f848d38a3eabe41032ea96, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/6ef2612327f1447fa33035bcac432061, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/31d9f0088f574d69b500094bf7146f68] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=33.3 K 2024-11-15T22:12:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-15T22:12:11,048 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting c1d0ec4ac1f848d38a3eabe41032ea96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731708728081 2024-11-15T22:12:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,049 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/14e12ed7675d460ab15ae5aee011892b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/afe19724cccd4db99c06ac0e39c36eab, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/980ad2107b5442c8850a5571dd7e6111] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=91.2 K 2024-11-15T22:12:11,049 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:11,049 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/14e12ed7675d460ab15ae5aee011892b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/afe19724cccd4db99c06ac0e39c36eab, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/980ad2107b5442c8850a5571dd7e6111] 2024-11-15T22:12:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,050 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14e12ed7675d460ab15ae5aee011892b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731708728081 2024-11-15T22:12:11,050 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ef2612327f1447fa33035bcac432061, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731708729262 2024-11-15T22:12:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T22:12:11,050 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:12:11,050 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting afe19724cccd4db99c06ac0e39c36eab, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731708729262 2024-11-15T22:12:11,050 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 31d9f0088f574d69b500094bf7146f68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731708730528 2024-11-15T22:12:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,051 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:12:11,051 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 980ad2107b5442c8850a5571dd7e6111, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731708730525 2024-11-15T22:12:11,051 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:12:11,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,066 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#B#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:11,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,067 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/5c783ebc160d4fc2a9aa1d7c63a953cd is 50, key is test_row_0/B:col10/1731708730531/Put/seqid=0 2024-11-15T22:12:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,071 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,099 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111565539dce76fc4355aa83d5b526e0101e_2a24e00e990dba09c516fd169bc5958d store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:11,101 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111565539dce76fc4355aa83d5b526e0101e_2a24e00e990dba09c516fd169bc5958d, store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:11,101 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111565539dce76fc4355aa83d5b526e0101e_2a24e00e990dba09c516fd169bc5958d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741961_1137 (size=12493) 2024-11-15T22:12:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,123 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/5c783ebc160d4fc2a9aa1d7c63a953cd as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/5c783ebc160d4fc2a9aa1d7c63a953cd 2024-11-15T22:12:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,133 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/B of 2a24e00e990dba09c516fd169bc5958d into 5c783ebc160d4fc2a9aa1d7c63a953cd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:11,133 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:11,133 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/B, priority=13, startTime=1731708731045; duration=0sec 2024-11-15T22:12:11,133 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:11,133 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:B 2024-11-15T22:12:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,133 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:11,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741962_1138 (size=4469) 2024-11-15T22:12:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,136 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:11,136 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/C is initiating minor compaction (all files) 2024-11-15T22:12:11,136 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/C in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:11,136 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/5cdc0d4bbfc143eda93acea988ceeafd, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/bc510524fc1e4babb15fa107eb22ffd6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/4e6ab6ebf86a4f19b98ec85cc877cbf9] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=33.3 K 2024-11-15T22:12:11,137 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cdc0d4bbfc143eda93acea988ceeafd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731708728081 2024-11-15T22:12:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,137 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting bc510524fc1e4babb15fa107eb22ffd6, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731708729262 2024-11-15T22:12:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,138 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e6ab6ebf86a4f19b98ec85cc877cbf9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731708730528 2024-11-15T22:12:11,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,141 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#A#compaction#112 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,142 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/5416bbc06db24199901d549c88586592 is 175, key is test_row_0/A:col10/1731708730531/Put/seqid=0 2024-11-15T22:12:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,154 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#C#compaction#113 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:11,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,154 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/af61a656f3f64383a6a5a019bf2855e7 is 50, key is test_row_0/C:col10/1731708730531/Put/seqid=0 2024-11-15T22:12:11,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T22:12:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,170 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:11,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T22:12:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741963_1139 (size=31447) 2024-11-15T22:12:11,202 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/5416bbc06db24199901d549c88586592 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/5416bbc06db24199901d549c88586592 2024-11-15T22:12:11,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T22:12:11,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:11,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:11,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:11,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:11,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:11,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741964_1140 (size=12493) 2024-11-15T22:12:11,211 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/A of 2a24e00e990dba09c516fd169bc5958d into 5416bbc06db24199901d549c88586592(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:11,211 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:11,211 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/A, priority=13, startTime=1731708731045; duration=0sec 2024-11-15T22:12:11,211 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:11,211 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:A 2024-11-15T22:12:11,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115811bd788d82b47ab92e49456ade6b77b_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708730890/Put/seqid=0 2024-11-15T22:12:11,222 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/af61a656f3f64383a6a5a019bf2855e7 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/af61a656f3f64383a6a5a019bf2855e7 2024-11-15T22:12:11,230 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/C of 2a24e00e990dba09c516fd169bc5958d into af61a656f3f64383a6a5a019bf2855e7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:11,230 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:11,230 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/C, priority=13, startTime=1731708731045; duration=0sec 2024-11-15T22:12:11,230 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:11,230 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:C 2024-11-15T22:12:11,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741965_1141 (size=17284) 2024-11-15T22:12:11,258 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708791251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,266 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115811bd788d82b47ab92e49456ade6b77b_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115811bd788d82b47ab92e49456ade6b77b_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:11,268 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/454c4e659c8a40918094ececd6472a01, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:11,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/454c4e659c8a40918094ececd6472a01 is 175, key is test_row_0/A:col10/1731708730890/Put/seqid=0 2024-11-15T22:12:11,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708791256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708791256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708791259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708791259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741966_1142 (size=48389) 2024-11-15T22:12:11,302 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/454c4e659c8a40918094ececd6472a01 2024-11-15T22:12:11,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/f4e94261dc3c4cf4b1414fd26027a376 is 50, key is test_row_0/B:col10/1731708730890/Put/seqid=0 2024-11-15T22:12:11,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741967_1143 (size=12151) 2024-11-15T22:12:11,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/f4e94261dc3c4cf4b1414fd26027a376 2024-11-15T22:12:11,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T22:12:11,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:11,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:11,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:11,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:11,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T22:12:11,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/e2caeaad18a94684a81f2e83c02128a1 is 50, key is test_row_0/C:col10/1731708730890/Put/seqid=0 2024-11-15T22:12:11,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708791376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708791377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708791377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708791377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708791377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741968_1144 (size=12151) 2024-11-15T22:12:11,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/e2caeaad18a94684a81f2e83c02128a1 2024-11-15T22:12:11,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/454c4e659c8a40918094ececd6472a01 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/454c4e659c8a40918094ececd6472a01 2024-11-15T22:12:11,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/454c4e659c8a40918094ececd6472a01, entries=250, sequenceid=176, filesize=47.3 K 2024-11-15T22:12:11,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/f4e94261dc3c4cf4b1414fd26027a376 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/f4e94261dc3c4cf4b1414fd26027a376 2024-11-15T22:12:11,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/f4e94261dc3c4cf4b1414fd26027a376, entries=150, sequenceid=176, filesize=11.9 K 2024-11-15T22:12:11,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/e2caeaad18a94684a81f2e83c02128a1 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/e2caeaad18a94684a81f2e83c02128a1 2024-11-15T22:12:11,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/e2caeaad18a94684a81f2e83c02128a1, entries=150, sequenceid=176, filesize=11.9 K 2024-11-15T22:12:11,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2a24e00e990dba09c516fd169bc5958d in 264ms, sequenceid=176, compaction requested=false 2024-11-15T22:12:11,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:11,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T22:12:11,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:11,519 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T22:12:11,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a964a95b1d914ca78a1814a099fadb5f_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708731255/Put/seqid=0 2024-11-15T22:12:11,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741969_1145 (size=12304) 2024-11-15T22:12:11,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,561 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a964a95b1d914ca78a1814a099fadb5f_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a964a95b1d914ca78a1814a099fadb5f_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:11,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/cd9dd4069867407599781411ce6697c7, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:11,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/cd9dd4069867407599781411ce6697c7 is 175, key is test_row_0/A:col10/1731708731255/Put/seqid=0 2024-11-15T22:12:11,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741970_1146 (size=31105) 2024-11-15T22:12:11,573 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/cd9dd4069867407599781411ce6697c7 2024-11-15T22:12:11,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/9237f462bdfb4ec6886cfd683307a9c0 is 50, key is test_row_0/B:col10/1731708731255/Put/seqid=0 2024-11-15T22:12:11,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:11,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:11,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741971_1147 (size=12151) 2024-11-15T22:12:11,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708791597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,604 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/9237f462bdfb4ec6886cfd683307a9c0 2024-11-15T22:12:11,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708791599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708791600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708791601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708791601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/1398c4e293654128ab31554715a40625 is 50, key is test_row_0/C:col10/1731708731255/Put/seqid=0 2024-11-15T22:12:11,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741972_1148 (size=12151) 2024-11-15T22:12:11,672 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/1398c4e293654128ab31554715a40625 2024-11-15T22:12:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T22:12:11,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/cd9dd4069867407599781411ce6697c7 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/cd9dd4069867407599781411ce6697c7 2024-11-15T22:12:11,690 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/cd9dd4069867407599781411ce6697c7, entries=150, sequenceid=202, filesize=30.4 K 2024-11-15T22:12:11,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/9237f462bdfb4ec6886cfd683307a9c0 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/9237f462bdfb4ec6886cfd683307a9c0 2024-11-15T22:12:11,700 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/9237f462bdfb4ec6886cfd683307a9c0, entries=150, sequenceid=202, filesize=11.9 K 2024-11-15T22:12:11,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/1398c4e293654128ab31554715a40625 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/1398c4e293654128ab31554715a40625 2024-11-15T22:12:11,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708791706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,713 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/1398c4e293654128ab31554715a40625, entries=150, sequenceid=202, filesize=11.9 K 2024-11-15T22:12:11,714 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 2a24e00e990dba09c516fd169bc5958d in 196ms, sequenceid=202, compaction requested=true 2024-11-15T22:12:11,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:11,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:11,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-15T22:12:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-15T22:12:11,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-15T22:12:11,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 665 msec 2024-11-15T22:12:11,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:11,722 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:11,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T22:12:11,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 674 msec 2024-11-15T22:12:11,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115312ca372b928481a8835a38e05c685d1_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708731592/Put/seqid=0 2024-11-15T22:12:11,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741973_1149 (size=14794) 2024-11-15T22:12:11,760 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:11,770 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115312ca372b928481a8835a38e05c685d1_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115312ca372b928481a8835a38e05c685d1_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:11,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708791758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,772 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/d27c63f5e1984e9f93f9e293ec3e0a55, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:11,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708791761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/d27c63f5e1984e9f93f9e293ec3e0a55 is 175, key is test_row_0/A:col10/1731708731592/Put/seqid=0 2024-11-15T22:12:11,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708791766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708791773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741974_1150 (size=39749) 2024-11-15T22:12:11,792 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/d27c63f5e1984e9f93f9e293ec3e0a55 2024-11-15T22:12:11,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/d2e1451635a14e56944ddedfe2e89a5e is 50, key is test_row_0/B:col10/1731708731592/Put/seqid=0 2024-11-15T22:12:11,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741975_1151 (size=12151) 2024-11-15T22:12:11,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/d2e1451635a14e56944ddedfe2e89a5e 2024-11-15T22:12:11,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708791877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708791877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708791886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708791886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/f17eb6c784ca4eec97b17f4cfc7f9c9b is 50, key is test_row_0/C:col10/1731708731592/Put/seqid=0 2024-11-15T22:12:11,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741976_1152 (size=12151) 2024-11-15T22:12:11,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/f17eb6c784ca4eec97b17f4cfc7f9c9b 2024-11-15T22:12:11,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/d27c63f5e1984e9f93f9e293ec3e0a55 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/d27c63f5e1984e9f93f9e293ec3e0a55 2024-11-15T22:12:11,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/d27c63f5e1984e9f93f9e293ec3e0a55, entries=200, sequenceid=218, filesize=38.8 K 2024-11-15T22:12:11,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/d2e1451635a14e56944ddedfe2e89a5e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/d2e1451635a14e56944ddedfe2e89a5e 2024-11-15T22:12:11,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708791917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:11,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:11,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:11,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/d2e1451635a14e56944ddedfe2e89a5e, entries=150, sequenceid=218, filesize=11.9 K 2024-11-15T22:12:11,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/f17eb6c784ca4eec97b17f4cfc7f9c9b as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/f17eb6c784ca4eec97b17f4cfc7f9c9b 2024-11-15T22:12:11,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/f17eb6c784ca4eec97b17f4cfc7f9c9b, entries=150, sequenceid=218, filesize=11.9 K 2024-11-15T22:12:11,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 2a24e00e990dba09c516fd169bc5958d in 221ms, sequenceid=218, compaction requested=true 2024-11-15T22:12:11,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:11,944 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:12:11,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:12:11,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:11,945 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:12:11,946 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150690 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:12:11,946 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/A is initiating minor compaction (all files) 2024-11-15T22:12:11,946 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/A in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:11,946 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/5416bbc06db24199901d549c88586592, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/454c4e659c8a40918094ececd6472a01, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/cd9dd4069867407599781411ce6697c7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/d27c63f5e1984e9f93f9e293ec3e0a55] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=147.2 K 2024-11-15T22:12:11,946 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:11,946 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/5416bbc06db24199901d549c88586592, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/454c4e659c8a40918094ececd6472a01, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/cd9dd4069867407599781411ce6697c7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/d27c63f5e1984e9f93f9e293ec3e0a55] 2024-11-15T22:12:11,947 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5416bbc06db24199901d549c88586592, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731708730528 2024-11-15T22:12:11,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:12:11,947 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 454c4e659c8a40918094ececd6472a01, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731708730890 2024-11-15T22:12:11,947 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:12:11,947 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/B is initiating minor compaction (all files) 2024-11-15T22:12:11,947 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/B in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:11,947 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/5c783ebc160d4fc2a9aa1d7c63a953cd, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/f4e94261dc3c4cf4b1414fd26027a376, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/9237f462bdfb4ec6886cfd683307a9c0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/d2e1451635a14e56944ddedfe2e89a5e] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=47.8 K 2024-11-15T22:12:11,948 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd9dd4069867407599781411ce6697c7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731708731244 2024-11-15T22:12:11,948 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c783ebc160d4fc2a9aa1d7c63a953cd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731708730528 2024-11-15T22:12:11,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:11,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:12:11,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:11,953 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting d27c63f5e1984e9f93f9e293ec3e0a55, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731708731592 2024-11-15T22:12:11,953 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting f4e94261dc3c4cf4b1414fd26027a376, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731708730890 2024-11-15T22:12:11,954 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 9237f462bdfb4ec6886cfd683307a9c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731708731244 2024-11-15T22:12:11,954 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting d2e1451635a14e56944ddedfe2e89a5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731708731592 2024-11-15T22:12:11,984 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#B#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:11,984 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/aa3b5263de4b451e9ce5e351fbb336ad is 50, key is test_row_0/B:col10/1731708731592/Put/seqid=0 2024-11-15T22:12:11,990 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:12,006 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115924b5e842dc947c29b175c1c080b7282_2a24e00e990dba09c516fd169bc5958d store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:12,008 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115924b5e842dc947c29b175c1c080b7282_2a24e00e990dba09c516fd169bc5958d, store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:12,009 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115924b5e842dc947c29b175c1c080b7282_2a24e00e990dba09c516fd169bc5958d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:12,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741978_1154 (size=4469) 2024-11-15T22:12:12,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741977_1153 (size=12629) 2024-11-15T22:12:12,069 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#A#compaction#124 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:12,070 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/612ba0cb7f394cb4a7c28d2f5148b177 is 175, key is test_row_0/A:col10/1731708731592/Put/seqid=0 2024-11-15T22:12:12,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741979_1155 (size=31583) 2024-11-15T22:12:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:12,092 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:12,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T22:12:12,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115314a5ac9fb6945de8471f0ee2c24d6ff_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708731754/Put/seqid=0 2024-11-15T22:12:12,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741980_1156 (size=14794) 2024-11-15T22:12:12,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708792165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T22:12:12,189 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:12,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708792166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:12,190 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:12:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:12,191 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-15T22:12:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T22:12:12,194 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:12:12,194 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:12:12,195 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:12:12,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708792175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:12,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708792183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:12,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:12,210 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115314a5ac9fb6945de8471f0ee2c24d6ff_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115314a5ac9fb6945de8471f0ee2c24d6ff_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:12,216 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/12b2608f10c248e3aafe67324e05d0ef, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:12,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/12b2608f10c248e3aafe67324e05d0ef is 175, key is test_row_0/A:col10/1731708731754/Put/seqid=0 2024-11-15T22:12:12,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708792252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:12,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741981_1157 (size=39749) 2024-11-15T22:12:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T22:12:12,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708792302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:12,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708792304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:12,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:12,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708792309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:12,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708792313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:12,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:12,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T22:12:12,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:12,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:12,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:12,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:12,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:12,487 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/aa3b5263de4b451e9ce5e351fbb336ad as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/aa3b5263de4b451e9ce5e351fbb336ad 2024-11-15T22:12:12,499 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/612ba0cb7f394cb4a7c28d2f5148b177 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/612ba0cb7f394cb4a7c28d2f5148b177 2024-11-15T22:12:12,500 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/B of 2a24e00e990dba09c516fd169bc5958d into aa3b5263de4b451e9ce5e351fbb336ad(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:12,501 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:12,501 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/B, priority=12, startTime=1731708731945; duration=0sec 2024-11-15T22:12:12,506 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:12,507 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:B 2024-11-15T22:12:12,507 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:12:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T22:12:12,510 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:12:12,510 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/C is initiating minor compaction (all files) 2024-11-15T22:12:12,510 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/C in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:12,510 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/af61a656f3f64383a6a5a019bf2855e7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/e2caeaad18a94684a81f2e83c02128a1, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/1398c4e293654128ab31554715a40625, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/f17eb6c784ca4eec97b17f4cfc7f9c9b] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=47.8 K 2024-11-15T22:12:12,511 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting af61a656f3f64383a6a5a019bf2855e7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731708730528 2024-11-15T22:12:12,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T22:12:12,514 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting e2caeaad18a94684a81f2e83c02128a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731708730890 2024-11-15T22:12:12,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:12,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:12,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:12,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:12,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:12,516 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/A of 2a24e00e990dba09c516fd169bc5958d into 612ba0cb7f394cb4a7c28d2f5148b177(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:12,516 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:12,517 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/A, priority=12, startTime=1731708731944; duration=0sec 2024-11-15T22:12:12,517 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:12,517 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:A 2024-11-15T22:12:12,517 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 1398c4e293654128ab31554715a40625, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731708731244 2024-11-15T22:12:12,518 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting f17eb6c784ca4eec97b17f4cfc7f9c9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731708731592 2024-11-15T22:12:12,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708792517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:12,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708792517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:12,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:12,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708792525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:12,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708792527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:12,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:12,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:12,550 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#C#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:12,551 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/9930bdda810f453ea5a3bbd6dace68f1 is 50, key is test_row_0/C:col10/1731708731592/Put/seqid=0 2024-11-15T22:12:12,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741982_1158 (size=12629) 2024-11-15T22:12:12,619 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/9930bdda810f453ea5a3bbd6dace68f1 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/9930bdda810f453ea5a3bbd6dace68f1 2024-11-15T22:12:12,637 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/C of 2a24e00e990dba09c516fd169bc5958d into 9930bdda810f453ea5a3bbd6dace68f1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:12,637 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:12,637 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/C, priority=12, startTime=1731708731948; duration=0sec 2024-11-15T22:12:12,638 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:12,639 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:C 2024-11-15T22:12:12,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T22:12:12,668 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/12b2608f10c248e3aafe67324e05d0ef 2024-11-15T22:12:12,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:12,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:12,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:12,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:12,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:12,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/da418c39dd044183bbf2fbebf7747994 is 50, key is test_row_0/B:col10/1731708731754/Put/seqid=0 2024-11-15T22:12:12,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741983_1159 (size=12151) 2024-11-15T22:12:12,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/da418c39dd044183bbf2fbebf7747994 2024-11-15T22:12:12,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:12,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708792769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:12,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/293a2748ebb9424a979c3e4eb6eab9a4 is 50, key is test_row_0/C:col10/1731708731754/Put/seqid=0 2024-11-15T22:12:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T22:12:12,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741984_1160 (size=12151) 2024-11-15T22:12:12,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T22:12:12,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:12,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:12,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:12,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:12,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:12,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708792837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:12,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708792837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:12,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:12,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:12,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708792837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:12,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708792846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:12,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:12,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:12,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T22:12:12,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:12,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:12,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:12,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:12,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:12,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:13,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T22:12:13,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:13,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:13,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:13,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:13,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:13,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:13,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/293a2748ebb9424a979c3e4eb6eab9a4 2024-11-15T22:12:13,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/12b2608f10c248e3aafe67324e05d0ef as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/12b2608f10c248e3aafe67324e05d0ef 2024-11-15T22:12:13,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/12b2608f10c248e3aafe67324e05d0ef, entries=200, sequenceid=240, filesize=38.8 K 2024-11-15T22:12:13,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/da418c39dd044183bbf2fbebf7747994 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/da418c39dd044183bbf2fbebf7747994 2024-11-15T22:12:13,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/da418c39dd044183bbf2fbebf7747994, entries=150, sequenceid=240, filesize=11.9 K 2024-11-15T22:12:13,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/293a2748ebb9424a979c3e4eb6eab9a4 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/293a2748ebb9424a979c3e4eb6eab9a4 2024-11-15T22:12:13,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/293a2748ebb9424a979c3e4eb6eab9a4, entries=150, sequenceid=240, filesize=11.9 K 2024-11-15T22:12:13,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 2a24e00e990dba09c516fd169bc5958d in 1177ms, sequenceid=240, compaction requested=false 2024-11-15T22:12:13,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:13,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T22:12:13,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:13,305 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T22:12:13,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153540accabec04f5883796d195d5767e8_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708732166/Put/seqid=0 2024-11-15T22:12:13,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T22:12:13,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741985_1161 (size=12304) 2024-11-15T22:12:13,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:13,340 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153540accabec04f5883796d195d5767e8_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153540accabec04f5883796d195d5767e8_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:13,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/46030bf4ce074b9b8dc2602f668dd3c6, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:13,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/46030bf4ce074b9b8dc2602f668dd3c6 is 175, key is test_row_0/A:col10/1731708732166/Put/seqid=0 2024-11-15T22:12:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:13,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:13,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741986_1162 (size=31105) 2024-11-15T22:12:13,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708793403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:13,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:13,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708793405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:13,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708793406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:13,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708793408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:13,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708793517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:13,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708793517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:13,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708793517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:13,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:13,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708793526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:13,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:13,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708793726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:13,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708793726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:13,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708793726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:13,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:13,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708793741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:13,775 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/46030bf4ce074b9b8dc2602f668dd3c6 2024-11-15T22:12:13,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/aeeaca18fbee492aad55e40ef9798ed3 is 50, key is test_row_0/B:col10/1731708732166/Put/seqid=0 2024-11-15T22:12:13,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:13,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708793786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:13,802 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_2' in 2a24e00e990dba09c516fd169bc5958d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:13,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:13,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741987_1163 (size=12151) 2024-11-15T22:12:13,808 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/aeeaca18fbee492aad55e40ef9798ed3 2024-11-15T22:12:13,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/3a9f1ed7c87642818a248ac91d840a28 is 50, key is test_row_0/C:col10/1731708732166/Put/seqid=0 2024-11-15T22:12:13,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741988_1164 (size=12151) 2024-11-15T22:12:14,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708794037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:14,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:14,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708794037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:14,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708794046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:14,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708794046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:14,253 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/3a9f1ed7c87642818a248ac91d840a28 2024-11-15T22:12:14,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/46030bf4ce074b9b8dc2602f668dd3c6 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/46030bf4ce074b9b8dc2602f668dd3c6 2024-11-15T22:12:14,267 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/46030bf4ce074b9b8dc2602f668dd3c6, entries=150, sequenceid=257, filesize=30.4 K 2024-11-15T22:12:14,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/aeeaca18fbee492aad55e40ef9798ed3 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/aeeaca18fbee492aad55e40ef9798ed3 2024-11-15T22:12:14,275 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/aeeaca18fbee492aad55e40ef9798ed3, entries=150, sequenceid=257, filesize=11.9 K 2024-11-15T22:12:14,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/3a9f1ed7c87642818a248ac91d840a28 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/3a9f1ed7c87642818a248ac91d840a28 2024-11-15T22:12:14,284 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/3a9f1ed7c87642818a248ac91d840a28, entries=150, sequenceid=257, filesize=11.9 K 2024-11-15T22:12:14,286 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 2a24e00e990dba09c516fd169bc5958d in 981ms, sequenceid=257, compaction requested=true 2024-11-15T22:12:14,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:14,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:14,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-15T22:12:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-15T22:12:14,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-15T22:12:14,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0950 sec 2024-11-15T22:12:14,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.1030 sec 2024-11-15T22:12:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T22:12:14,336 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:14,337 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:12:14,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-15T22:12:14,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T22:12:14,340 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:12:14,341 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:12:14,341 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:12:14,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T22:12:14,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T22:12:14,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:14,496 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T22:12:14,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115163e776ed7bc4c02ae8c83a883fa061d_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708733402/Put/seqid=0 2024-11-15T22:12:14,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741989_1165 (size=12454) 2024-11-15T22:12:14,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:14,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:14,567 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115163e776ed7bc4c02ae8c83a883fa061d_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115163e776ed7bc4c02ae8c83a883fa061d_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:14,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/b2838c5d32d64eec8b36c71070530809, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:14,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/b2838c5d32d64eec8b36c71070530809 is 175, key is test_row_0/A:col10/1731708733402/Put/seqid=0 2024-11-15T22:12:14,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708794584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:14,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:14,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708794585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:14,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708794589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:14,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:14,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708794594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:14,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741990_1166 (size=31255) 2024-11-15T22:12:14,602 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/b2838c5d32d64eec8b36c71070530809 2024-11-15T22:12:14,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/ed3ea6388bde43fe9b6c1c3f56ff14c5 is 50, key is test_row_0/B:col10/1731708733402/Put/seqid=0 2024-11-15T22:12:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T22:12:14,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741991_1167 (size=12301) 2024-11-15T22:12:14,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708794697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:14,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:14,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708794697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:14,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:14,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:14,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708794706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:14,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708794706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:14,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708794907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:14,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708794907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:14,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:14,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:14,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708794918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:14,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708794918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:14,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T22:12:15,080 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/ed3ea6388bde43fe9b6c1c3f56ff14c5 2024-11-15T22:12:15,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/818d7c09176846989a531bf9a9169143 is 50, key is test_row_0/C:col10/1731708733402/Put/seqid=0 2024-11-15T22:12:15,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741992_1168 (size=12301) 2024-11-15T22:12:15,100 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/818d7c09176846989a531bf9a9169143 2024-11-15T22:12:15,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/b2838c5d32d64eec8b36c71070530809 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/b2838c5d32d64eec8b36c71070530809 2024-11-15T22:12:15,113 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/b2838c5d32d64eec8b36c71070530809, entries=150, sequenceid=279, filesize=30.5 K 2024-11-15T22:12:15,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/ed3ea6388bde43fe9b6c1c3f56ff14c5 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/ed3ea6388bde43fe9b6c1c3f56ff14c5 2024-11-15T22:12:15,120 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/ed3ea6388bde43fe9b6c1c3f56ff14c5, entries=150, sequenceid=279, filesize=12.0 K 2024-11-15T22:12:15,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/818d7c09176846989a531bf9a9169143 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/818d7c09176846989a531bf9a9169143 2024-11-15T22:12:15,127 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/818d7c09176846989a531bf9a9169143, entries=150, sequenceid=279, filesize=12.0 K 2024-11-15T22:12:15,128 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 2a24e00e990dba09c516fd169bc5958d in 633ms, sequenceid=279, compaction requested=true 2024-11-15T22:12:15,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:15,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:15,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-15T22:12:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-15T22:12:15,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-15T22:12:15,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 789 msec 2024-11-15T22:12:15,136 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 797 msec 2024-11-15T22:12:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:15,237 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:15,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T22:12:15,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152c978e7a159041eab7a195ca85fa50a5_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708735235/Put/seqid=0 2024-11-15T22:12:15,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708795260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:15,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708795261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:15,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708795265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:15,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:15,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741993_1169 (size=14994) 2024-11-15T22:12:15,272 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:15,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708795267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:15,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:15,280 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152c978e7a159041eab7a195ca85fa50a5_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152c978e7a159041eab7a195ca85fa50a5_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:15,281 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/d23c70e7f0c04580aa7270ef3eb804e6, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:15,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/d23c70e7f0c04580aa7270ef3eb804e6 is 175, key is test_row_0/A:col10/1731708735235/Put/seqid=0 2024-11-15T22:12:15,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741994_1170 (size=39949) 2024-11-15T22:12:15,294 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/d23c70e7f0c04580aa7270ef3eb804e6 2024-11-15T22:12:15,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/515fe90653ff4693a518b372fec0ed77 is 50, key is test_row_0/B:col10/1731708735235/Put/seqid=0 2024-11-15T22:12:15,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741995_1171 (size=12301) 2024-11-15T22:12:15,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/515fe90653ff4693a518b372fec0ed77 2024-11-15T22:12:15,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/e88821a670a54466814df8ec96d9df9b is 50, key is test_row_0/C:col10/1731708735235/Put/seqid=0 2024-11-15T22:12:15,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741996_1172 (size=12301) 2024-11-15T22:12:15,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708795377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:15,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:15,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708795381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708795381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:15,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:15,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708795387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:15,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:15,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T22:12:15,487 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:15,487 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:12:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-15T22:12:15,490 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:12:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T22:12:15,491 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:12:15,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:12:15,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708795586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T22:12:15,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708795596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:15,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:15,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708795596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:15,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:15,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708795597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:15,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:15,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:15,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T22:12:15,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:15,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:15,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:15,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:15,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:15,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/e88821a670a54466814df8ec96d9df9b 2024-11-15T22:12:15,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/d23c70e7f0c04580aa7270ef3eb804e6 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/d23c70e7f0c04580aa7270ef3eb804e6 2024-11-15T22:12:15,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/d23c70e7f0c04580aa7270ef3eb804e6, entries=200, sequenceid=298, filesize=39.0 K 2024-11-15T22:12:15,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/515fe90653ff4693a518b372fec0ed77 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/515fe90653ff4693a518b372fec0ed77 2024-11-15T22:12:15,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/515fe90653ff4693a518b372fec0ed77, entries=150, sequenceid=298, filesize=12.0 K 2024-11-15T22:12:15,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/e88821a670a54466814df8ec96d9df9b as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/e88821a670a54466814df8ec96d9df9b 2024-11-15T22:12:15,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/e88821a670a54466814df8ec96d9df9b, entries=150, sequenceid=298, filesize=12.0 K 2024-11-15T22:12:15,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 2a24e00e990dba09c516fd169bc5958d in 536ms, sequenceid=298, compaction requested=true 2024-11-15T22:12:15,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:15,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:12:15,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:15,773 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T22:12:15,774 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T22:12:15,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:12:15,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:15,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:12:15,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:15,776 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 173641 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T22:12:15,776 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61533 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T22:12:15,776 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/A is initiating minor compaction (all files) 2024-11-15T22:12:15,776 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/B is initiating minor compaction (all files) 2024-11-15T22:12:15,776 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/A in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:15,776 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/B in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:15,776 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/aa3b5263de4b451e9ce5e351fbb336ad, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/da418c39dd044183bbf2fbebf7747994, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/aeeaca18fbee492aad55e40ef9798ed3, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/ed3ea6388bde43fe9b6c1c3f56ff14c5, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/515fe90653ff4693a518b372fec0ed77] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=60.1 K 2024-11-15T22:12:15,776 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/612ba0cb7f394cb4a7c28d2f5148b177, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/12b2608f10c248e3aafe67324e05d0ef, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/46030bf4ce074b9b8dc2602f668dd3c6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/b2838c5d32d64eec8b36c71070530809, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/d23c70e7f0c04580aa7270ef3eb804e6] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=169.6 K 2024-11-15T22:12:15,776 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:15,776 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/612ba0cb7f394cb4a7c28d2f5148b177, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/12b2608f10c248e3aafe67324e05d0ef, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/46030bf4ce074b9b8dc2602f668dd3c6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/b2838c5d32d64eec8b36c71070530809, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/d23c70e7f0c04580aa7270ef3eb804e6] 2024-11-15T22:12:15,776 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting aa3b5263de4b451e9ce5e351fbb336ad, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731708731592 2024-11-15T22:12:15,777 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 612ba0cb7f394cb4a7c28d2f5148b177, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731708731592 2024-11-15T22:12:15,777 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting da418c39dd044183bbf2fbebf7747994, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731708731754 2024-11-15T22:12:15,777 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12b2608f10c248e3aafe67324e05d0ef, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731708731754 2024-11-15T22:12:15,777 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting aeeaca18fbee492aad55e40ef9798ed3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731708732108 2024-11-15T22:12:15,777 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46030bf4ce074b9b8dc2602f668dd3c6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731708732108 2024-11-15T22:12:15,778 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting ed3ea6388bde43fe9b6c1c3f56ff14c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731708733402 2024-11-15T22:12:15,778 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2838c5d32d64eec8b36c71070530809, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731708733402 2024-11-15T22:12:15,778 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting d23c70e7f0c04580aa7270ef3eb804e6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731708735227 2024-11-15T22:12:15,778 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 515fe90653ff4693a518b372fec0ed77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731708735228 2024-11-15T22:12:15,791 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:15,794 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#B#compaction#139 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:15,795 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/b5d978be02d4495081bd69c94fdb144a is 50, key is test_row_0/B:col10/1731708735235/Put/seqid=0 2024-11-15T22:12:15,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T22:12:15,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:15,800 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T22:12:15,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T22:12:15,810 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111526512517f52c47beb1913c7741402248_2a24e00e990dba09c516fd169bc5958d store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:15,813 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111526512517f52c47beb1913c7741402248_2a24e00e990dba09c516fd169bc5958d, store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:15,813 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111526512517f52c47beb1913c7741402248_2a24e00e990dba09c516fd169bc5958d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:15,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:15,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741997_1173 (size=12949) 2024-11-15T22:12:15,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153c8d0fc05b12409ba315b5b923299aba_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708735263/Put/seqid=0 2024-11-15T22:12:15,862 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/b5d978be02d4495081bd69c94fdb144a as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/b5d978be02d4495081bd69c94fdb144a 2024-11-15T22:12:15,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741999_1175 (size=12454) 2024-11-15T22:12:15,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741998_1174 (size=4469) 2024-11-15T22:12:15,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:15,885 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153c8d0fc05b12409ba315b5b923299aba_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153c8d0fc05b12409ba315b5b923299aba_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:15,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/d9e99cf6b3b84c58b0750b93d10e9c3b, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:15,888 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/B of 2a24e00e990dba09c516fd169bc5958d into b5d978be02d4495081bd69c94fdb144a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:15,888 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:15,888 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/B, priority=11, startTime=1731708735773; duration=0sec 2024-11-15T22:12:15,888 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:15,888 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:B 2024-11-15T22:12:15,888 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T22:12:15,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/d9e99cf6b3b84c58b0750b93d10e9c3b is 175, key is test_row_0/A:col10/1731708735263/Put/seqid=0 2024-11-15T22:12:15,891 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61533 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T22:12:15,891 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/C is initiating minor compaction (all files) 2024-11-15T22:12:15,891 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/C in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:15,892 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/9930bdda810f453ea5a3bbd6dace68f1, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/293a2748ebb9424a979c3e4eb6eab9a4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/3a9f1ed7c87642818a248ac91d840a28, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/818d7c09176846989a531bf9a9169143, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/e88821a670a54466814df8ec96d9df9b] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=60.1 K 2024-11-15T22:12:15,892 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 9930bdda810f453ea5a3bbd6dace68f1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731708731592 2024-11-15T22:12:15,893 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 293a2748ebb9424a979c3e4eb6eab9a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731708731754 2024-11-15T22:12:15,897 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a9f1ed7c87642818a248ac91d840a28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731708732108 2024-11-15T22:12:15,898 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 818d7c09176846989a531bf9a9169143, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731708733402 2024-11-15T22:12:15,898 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting e88821a670a54466814df8ec96d9df9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731708735228 2024-11-15T22:12:15,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708795906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:15,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:15,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708795909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:15,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:15,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742000_1176 (size=31255) 2024-11-15T22:12:15,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:15,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708795910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:15,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:15,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708795911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:15,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:15,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708795913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:15,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:15,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:15,932 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:15,933 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/afc8f63a823141d28696a35e528e5c06 is 50, key is test_row_0/C:col10/1731708735235/Put/seqid=0 2024-11-15T22:12:15,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742001_1177 (size=12949) 2024-11-15T22:12:16,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:16,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708796016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:16,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:16,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708796027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:16,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708796027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:16,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708796027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:16,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T22:12:16,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708796226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:16,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708796246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:16,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708796246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708796246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:16,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:16,267 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#A#compaction#138 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:16,268 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/12d9edffd35f4e2695fc45b5a373e025 is 175, key is test_row_0/A:col10/1731708735235/Put/seqid=0 2024-11-15T22:12:16,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742002_1178 (size=31903) 2024-11-15T22:12:16,320 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/d9e99cf6b3b84c58b0750b93d10e9c3b 2024-11-15T22:12:16,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/67c871c841794d8da3549730bc4e7cd8 is 50, key is test_row_0/B:col10/1731708735263/Put/seqid=0 2024-11-15T22:12:16,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742003_1179 (size=12301) 2024-11-15T22:12:16,340 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/67c871c841794d8da3549730bc4e7cd8 2024-11-15T22:12:16,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/f25e8d1894124ddbad8ef231e12d0f74 is 50, key is test_row_0/C:col10/1731708735263/Put/seqid=0 2024-11-15T22:12:16,354 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/afc8f63a823141d28696a35e528e5c06 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/afc8f63a823141d28696a35e528e5c06 2024-11-15T22:12:16,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742004_1180 (size=12301) 2024-11-15T22:12:16,361 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/f25e8d1894124ddbad8ef231e12d0f74 2024-11-15T22:12:16,364 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/C of 2a24e00e990dba09c516fd169bc5958d into afc8f63a823141d28696a35e528e5c06(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:16,364 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:16,365 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/C, priority=11, startTime=1731708735774; duration=0sec 2024-11-15T22:12:16,365 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:16,365 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:C 2024-11-15T22:12:16,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/d9e99cf6b3b84c58b0750b93d10e9c3b as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/d9e99cf6b3b84c58b0750b93d10e9c3b 2024-11-15T22:12:16,391 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/d9e99cf6b3b84c58b0750b93d10e9c3b, entries=150, sequenceid=315, filesize=30.5 K 2024-11-15T22:12:16,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/67c871c841794d8da3549730bc4e7cd8 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/67c871c841794d8da3549730bc4e7cd8 2024-11-15T22:12:16,399 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/67c871c841794d8da3549730bc4e7cd8, entries=150, sequenceid=315, filesize=12.0 K 2024-11-15T22:12:16,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/f25e8d1894124ddbad8ef231e12d0f74 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/f25e8d1894124ddbad8ef231e12d0f74 2024-11-15T22:12:16,410 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/f25e8d1894124ddbad8ef231e12d0f74, entries=150, sequenceid=315, filesize=12.0 K 2024-11-15T22:12:16,411 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 2a24e00e990dba09c516fd169bc5958d in 611ms, sequenceid=315, compaction requested=false 2024-11-15T22:12:16,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:16,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:16,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-15T22:12:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-15T22:12:16,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-15T22:12:16,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 922 msec 2024-11-15T22:12:16,418 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 929 msec 2024-11-15T22:12:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:16,421 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:16,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-15T22:12:16,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115678f1303c39b472fa3a4a2a131a283ca_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708735905/Put/seqid=0 2024-11-15T22:12:16,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742005_1181 (size=12454) 2024-11-15T22:12:16,442 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:16,454 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115678f1303c39b472fa3a4a2a131a283ca_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115678f1303c39b472fa3a4a2a131a283ca_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:16,456 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/a9bbbf08c3d04cf7adf43dabf233ee0d, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:16,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/a9bbbf08c3d04cf7adf43dabf233ee0d is 175, key is test_row_0/A:col10/1731708735905/Put/seqid=0 2024-11-15T22:12:16,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742006_1182 (size=31255) 2024-11-15T22:12:16,462 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=337, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/a9bbbf08c3d04cf7adf43dabf233ee0d 2024-11-15T22:12:16,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/361386c5e9e244499f1cacb1d7b1aea9 is 50, key is test_row_0/B:col10/1731708735905/Put/seqid=0 2024-11-15T22:12:16,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742007_1183 (size=12301) 2024-11-15T22:12:16,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/361386c5e9e244499f1cacb1d7b1aea9 2024-11-15T22:12:16,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/bf6b147a3339400b87bdb9810af7decb is 50, key is test_row_0/C:col10/1731708735905/Put/seqid=0 2024-11-15T22:12:16,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708796510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:16,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:16,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742008_1184 (size=12301) 2024-11-15T22:12:16,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/bf6b147a3339400b87bdb9810af7decb 2024-11-15T22:12:16,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708796536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:16,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:16,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/a9bbbf08c3d04cf7adf43dabf233ee0d as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/a9bbbf08c3d04cf7adf43dabf233ee0d 2024-11-15T22:12:16,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/a9bbbf08c3d04cf7adf43dabf233ee0d, entries=150, sequenceid=337, filesize=30.5 K 2024-11-15T22:12:16,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/361386c5e9e244499f1cacb1d7b1aea9 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/361386c5e9e244499f1cacb1d7b1aea9 2024-11-15T22:12:16,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:16,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/361386c5e9e244499f1cacb1d7b1aea9, entries=150, sequenceid=337, filesize=12.0 K 2024-11-15T22:12:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708796557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:16,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:16,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708796557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:16,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/bf6b147a3339400b87bdb9810af7decb as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/bf6b147a3339400b87bdb9810af7decb 2024-11-15T22:12:16,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:16,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:16,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708796557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:16,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:16,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/bf6b147a3339400b87bdb9810af7decb, entries=150, sequenceid=337, filesize=12.0 K 2024-11-15T22:12:16,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 2a24e00e990dba09c516fd169bc5958d in 149ms, sequenceid=337, compaction requested=true 2024-11-15T22:12:16,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:16,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:12:16,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:16,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:12:16,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:16,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:12:16,570 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 5 compacting, 2 eligible, 16 blocking 2024-11-15T22:12:16,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:12:16,571 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T22:12:16,571 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T22:12:16,571 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. because compaction request was cancelled 2024-11-15T22:12:16,571 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:A 2024-11-15T22:12:16,571 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:16,573 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:16,573 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/B is initiating minor compaction (all files) 2024-11-15T22:12:16,573 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/B in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:16,573 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/b5d978be02d4495081bd69c94fdb144a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/67c871c841794d8da3549730bc4e7cd8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/361386c5e9e244499f1cacb1d7b1aea9] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=36.7 K 2024-11-15T22:12:16,574 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting b5d978be02d4495081bd69c94fdb144a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731708735228 2024-11-15T22:12:16,574 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 67c871c841794d8da3549730bc4e7cd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731708735249 2024-11-15T22:12:16,574 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 361386c5e9e244499f1cacb1d7b1aea9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731708735905 2024-11-15T22:12:16,583 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#B#compaction#147 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:16,583 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/6cdc951b02924907b846fb4cbe3a0bce is 50, key is test_row_0/B:col10/1731708735905/Put/seqid=0 2024-11-15T22:12:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T22:12:16,627 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:16,628 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:12:16,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742009_1185 (size=13051) 2024-11-15T22:12:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-15T22:12:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T22:12:16,633 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:12:16,634 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:16,634 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:12:16,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T22:12:16,634 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:12:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:16,638 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/6cdc951b02924907b846fb4cbe3a0bce as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/6cdc951b02924907b846fb4cbe3a0bce 2024-11-15T22:12:16,649 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/B of 2a24e00e990dba09c516fd169bc5958d into 6cdc951b02924907b846fb4cbe3a0bce(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:16,649 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:16,649 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/B, priority=13, startTime=1731708736570; duration=0sec 2024-11-15T22:12:16,649 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:16,649 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:B 2024-11-15T22:12:16,649 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:16,651 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:16,651 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/C is initiating minor compaction (all files) 2024-11-15T22:12:16,651 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/C in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:16,651 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/afc8f63a823141d28696a35e528e5c06, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/f25e8d1894124ddbad8ef231e12d0f74, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/bf6b147a3339400b87bdb9810af7decb] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=36.7 K 2024-11-15T22:12:16,651 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting afc8f63a823141d28696a35e528e5c06, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731708735228 2024-11-15T22:12:16,652 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting f25e8d1894124ddbad8ef231e12d0f74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731708735249 2024-11-15T22:12:16,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157985b1f41f304df0a76390d3d3792eda_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708736504/Put/seqid=0 2024-11-15T22:12:16,652 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting bf6b147a3339400b87bdb9810af7decb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731708735905 2024-11-15T22:12:16,688 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#C#compaction#149 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:16,688 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/37b58779b06e45ebbf2b6809e5a4e028 is 50, key is test_row_0/C:col10/1731708735905/Put/seqid=0 2024-11-15T22:12:16,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742010_1186 (size=14994) 2024-11-15T22:12:16,693 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:16,707 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157985b1f41f304df0a76390d3d3792eda_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157985b1f41f304df0a76390d3d3792eda_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:16,708 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/191a0139bf1b435884b0e060b2db73bb, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:16,708 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/12d9edffd35f4e2695fc45b5a373e025 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/12d9edffd35f4e2695fc45b5a373e025 2024-11-15T22:12:16,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/191a0139bf1b435884b0e060b2db73bb is 175, key is test_row_0/A:col10/1731708736504/Put/seqid=0 2024-11-15T22:12:16,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742011_1187 (size=13051) 2024-11-15T22:12:16,722 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/A of 2a24e00e990dba09c516fd169bc5958d into 12d9edffd35f4e2695fc45b5a373e025(size=31.2 K), total size for store is 92.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:16,722 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:16,722 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/A, priority=11, startTime=1731708735773; duration=0sec 2024-11-15T22:12:16,722 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:16,722 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:A 2024-11-15T22:12:16,722 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/37b58779b06e45ebbf2b6809e5a4e028 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/37b58779b06e45ebbf2b6809e5a4e028 2024-11-15T22:12:16,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T22:12:16,738 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/C of 2a24e00e990dba09c516fd169bc5958d into 37b58779b06e45ebbf2b6809e5a4e028(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:16,738 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:16,738 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/C, priority=13, startTime=1731708736570; duration=0sec 2024-11-15T22:12:16,738 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:16,738 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:C 2024-11-15T22:12:16,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742012_1188 (size=39949) 2024-11-15T22:12:16,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T22:12:16,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:16,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:16,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:16,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:16,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:16,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708796809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:16,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708796927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:16,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:16,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:16,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T22:12:16,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T22:12:16,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:16,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:16,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:16,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:16,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:17,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708797046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:17,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:17,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:17,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:17,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708797076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:17,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708797077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:17,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:17,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708797077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:17,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:17,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:17,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:17,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:17,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:17,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:17,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:17,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:17,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T22:12:17,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:17,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:17,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:17,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:17,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:17,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708797136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:17,145 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/191a0139bf1b435884b0e060b2db73bb 2024-11-15T22:12:17,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/ffdf0ec9832a4182b85a1ea4c9978ad9 is 50, key is test_row_0/B:col10/1731708736504/Put/seqid=0 2024-11-15T22:12:17,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742013_1189 (size=12301) 2024-11-15T22:12:17,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/ffdf0ec9832a4182b85a1ea4c9978ad9 2024-11-15T22:12:17,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/3120e16763404c2bb51f038f1a37d435 is 50, key is test_row_0/C:col10/1731708736504/Put/seqid=0 2024-11-15T22:12:17,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742014_1190 (size=12301) 2024-11-15T22:12:17,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T22:12:17,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T22:12:17,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:17,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:17,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:17,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:17,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:17,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T22:12:17,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:17,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:17,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:17,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:17,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:17,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:17,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:17,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708797446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:17,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:17,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:17,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T22:12:17,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:17,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:17,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:17,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:17,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:17,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:17,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/3120e16763404c2bb51f038f1a37d435 2024-11-15T22:12:17,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/191a0139bf1b435884b0e060b2db73bb as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/191a0139bf1b435884b0e060b2db73bb 2024-11-15T22:12:17,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/191a0139bf1b435884b0e060b2db73bb, entries=200, sequenceid=354, filesize=39.0 K 2024-11-15T22:12:17,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/ffdf0ec9832a4182b85a1ea4c9978ad9 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/ffdf0ec9832a4182b85a1ea4c9978ad9 2024-11-15T22:12:17,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/ffdf0ec9832a4182b85a1ea4c9978ad9, entries=150, sequenceid=354, filesize=12.0 K 2024-11-15T22:12:17,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/3120e16763404c2bb51f038f1a37d435 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/3120e16763404c2bb51f038f1a37d435 2024-11-15T22:12:17,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/3120e16763404c2bb51f038f1a37d435, entries=150, sequenceid=354, filesize=12.0 K 2024-11-15T22:12:17,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 2a24e00e990dba09c516fd169bc5958d in 1030ms, sequenceid=354, compaction requested=true 2024-11-15T22:12:17,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:17,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:12:17,665 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:12:17,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:17,665 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T22:12:17,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:12:17,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:17,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:12:17,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:17,666 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T22:12:17,666 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T22:12:17,666 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. because compaction request was cancelled 2024-11-15T22:12:17,666 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:B 2024-11-15T22:12:17,666 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T22:12:17,667 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:12:17,667 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/A is initiating minor compaction (all files) 2024-11-15T22:12:17,667 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T22:12:17,667 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/A in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:17,667 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T22:12:17,667 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. because compaction request was cancelled 2024-11-15T22:12:17,667 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:C 2024-11-15T22:12:17,668 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/12d9edffd35f4e2695fc45b5a373e025, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/d9e99cf6b3b84c58b0750b93d10e9c3b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/a9bbbf08c3d04cf7adf43dabf233ee0d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/191a0139bf1b435884b0e060b2db73bb] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=131.2 K 2024-11-15T22:12:17,668 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:17,668 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/12d9edffd35f4e2695fc45b5a373e025, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/d9e99cf6b3b84c58b0750b93d10e9c3b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/a9bbbf08c3d04cf7adf43dabf233ee0d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/191a0139bf1b435884b0e060b2db73bb] 2024-11-15T22:12:17,668 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12d9edffd35f4e2695fc45b5a373e025, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731708735228 2024-11-15T22:12:17,669 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9e99cf6b3b84c58b0750b93d10e9c3b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731708735249 2024-11-15T22:12:17,669 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9bbbf08c3d04cf7adf43dabf233ee0d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731708735905 2024-11-15T22:12:17,670 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 191a0139bf1b435884b0e060b2db73bb, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731708736485 2024-11-15T22:12:17,679 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:17,682 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115986a99bdda7b4aaf8236e117267acdc0_2a24e00e990dba09c516fd169bc5958d store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:17,685 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115986a99bdda7b4aaf8236e117267acdc0_2a24e00e990dba09c516fd169bc5958d, store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:17,685 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115986a99bdda7b4aaf8236e117267acdc0_2a24e00e990dba09c516fd169bc5958d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:17,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742015_1191 (size=4469) 2024-11-15T22:12:17,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T22:12:17,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:17,731 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T22:12:17,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111510b6e4ce0a2e4b55ad136383de49d746_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708736705/Put/seqid=0 2024-11-15T22:12:17,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T22:12:17,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742016_1192 (size=12454) 2024-11-15T22:12:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:17,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:18,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708798037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:18,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708798066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:18,069 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_1' in 2a24e00e990dba09c516fd169bc5958d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:18,097 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#A#compaction#152 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:18,098 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/0be2dc7388294e4791b7f667598ff367 is 175, key is test_row_0/A:col10/1731708736504/Put/seqid=0 2024-11-15T22:12:18,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742017_1193 (size=32039) 2024-11-15T22:12:18,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708798096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:18,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708798097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:18,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708798097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:18,103 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_2' in 2a24e00e990dba09c516fd169bc5958d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:18,104 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_2' in 2a24e00e990dba09c516fd169bc5958d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:18,104 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_0' in 2a24e00e990dba09c516fd169bc5958d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:18,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708798146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:18,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:18,172 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111510b6e4ce0a2e4b55ad136383de49d746_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111510b6e4ce0a2e4b55ad136383de49d746_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:18,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/efce8d996e45455c8d4b8f1f6c022c96, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:18,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/efce8d996e45455c8d4b8f1f6c022c96 is 175, key is test_row_0/A:col10/1731708736705/Put/seqid=0 2024-11-15T22:12:18,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742018_1194 (size=31255) 2024-11-15T22:12:18,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:18,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708798370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:18,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:18,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:18,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:18,511 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/0be2dc7388294e4791b7f667598ff367 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/0be2dc7388294e4791b7f667598ff367 2024-11-15T22:12:18,517 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/A of 2a24e00e990dba09c516fd169bc5958d into 0be2dc7388294e4791b7f667598ff367(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:18,517 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:18,517 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/A, priority=12, startTime=1731708737665; duration=0sec 2024-11-15T22:12:18,517 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:18,517 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:A 2024-11-15T22:12:18,582 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=376, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/efce8d996e45455c8d4b8f1f6c022c96 2024-11-15T22:12:18,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/9885ad4b9fb34fd898497bbb78e791f2 is 50, key is test_row_0/B:col10/1731708736705/Put/seqid=0 2024-11-15T22:12:18,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742019_1195 (size=12301) 2024-11-15T22:12:18,603 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/9885ad4b9fb34fd898497bbb78e791f2 2024-11-15T22:12:18,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/48f9b70d1b94480fb0a3fc3ac008a95c is 50, key is test_row_0/C:col10/1731708736705/Put/seqid=0 2024-11-15T22:12:18,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742020_1196 (size=12301) 2024-11-15T22:12:18,625 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/48f9b70d1b94480fb0a3fc3ac008a95c 2024-11-15T22:12:18,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/efce8d996e45455c8d4b8f1f6c022c96 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/efce8d996e45455c8d4b8f1f6c022c96 2024-11-15T22:12:18,636 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/efce8d996e45455c8d4b8f1f6c022c96, entries=150, sequenceid=376, filesize=30.5 K 2024-11-15T22:12:18,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/9885ad4b9fb34fd898497bbb78e791f2 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/9885ad4b9fb34fd898497bbb78e791f2 2024-11-15T22:12:18,645 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/9885ad4b9fb34fd898497bbb78e791f2, entries=150, sequenceid=376, filesize=12.0 K 2024-11-15T22:12:18,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/48f9b70d1b94480fb0a3fc3ac008a95c as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/48f9b70d1b94480fb0a3fc3ac008a95c 2024-11-15T22:12:18,654 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/48f9b70d1b94480fb0a3fc3ac008a95c, entries=150, sequenceid=376, filesize=12.0 K 2024-11-15T22:12:18,655 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 2a24e00e990dba09c516fd169bc5958d in 924ms, sequenceid=376, compaction requested=true 2024-11-15T22:12:18,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:18,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:18,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-15T22:12:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-15T22:12:18,660 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-15T22:12:18,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0240 sec 2024-11-15T22:12:18,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.0330 sec 2024-11-15T22:12:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:18,691 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:18,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-15T22:12:18,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111518d2c88fc0564569881b791d594a89e7_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708738690/Put/seqid=0 2024-11-15T22:12:18,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742021_1197 (size=12454) 2024-11-15T22:12:18,736 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:18,746 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111518d2c88fc0564569881b791d594a89e7_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111518d2c88fc0564569881b791d594a89e7_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:18,747 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/1ffee492a4da48ffaa11d2a5160269c9, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:18,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/1ffee492a4da48ffaa11d2a5160269c9 is 175, key is test_row_0/A:col10/1731708738690/Put/seqid=0 2024-11-15T22:12:18,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742022_1198 (size=31255) 2024-11-15T22:12:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T22:12:18,778 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:18,778 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:12:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-15T22:12:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T22:12:18,782 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:12:18,783 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:12:18,783 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:12:18,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708798791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T22:12:18,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708798906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:18,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:18,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:18,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T22:12:18,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:18,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:18,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:18,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:18,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:19,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T22:12:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T22:12:19,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:19,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:19,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:19,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:19,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:19,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708799127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:19,171 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=395, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/1ffee492a4da48ffaa11d2a5160269c9 2024-11-15T22:12:19,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/aa28a6ed19ba4537b1517c34cdc56917 is 50, key is test_row_0/B:col10/1731708738690/Put/seqid=0 2024-11-15T22:12:19,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742023_1199 (size=12301) 2024-11-15T22:12:19,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T22:12:19,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:19,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:19,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:19,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:19,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:19,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:19,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T22:12:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T22:12:19,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:19,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:19,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:19,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:19,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:19,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708799447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:19,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:19,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:19,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:19,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T22:12:19,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:19,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:19,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:19,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:19,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:19,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:19,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/aa28a6ed19ba4537b1517c34cdc56917 2024-11-15T22:12:19,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/2f3e12c5a865410582e5c87816f6c3ac is 50, key is test_row_0/C:col10/1731708738690/Put/seqid=0 2024-11-15T22:12:19,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742024_1200 (size=12301) 2024-11-15T22:12:19,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/2f3e12c5a865410582e5c87816f6c3ac 2024-11-15T22:12:19,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/1ffee492a4da48ffaa11d2a5160269c9 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/1ffee492a4da48ffaa11d2a5160269c9 2024-11-15T22:12:19,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/1ffee492a4da48ffaa11d2a5160269c9, entries=150, sequenceid=395, filesize=30.5 K 2024-11-15T22:12:19,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/aa28a6ed19ba4537b1517c34cdc56917 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/aa28a6ed19ba4537b1517c34cdc56917 2024-11-15T22:12:19,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/aa28a6ed19ba4537b1517c34cdc56917, entries=150, sequenceid=395, filesize=12.0 K 2024-11-15T22:12:19,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/2f3e12c5a865410582e5c87816f6c3ac as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/2f3e12c5a865410582e5c87816f6c3ac 2024-11-15T22:12:19,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/2f3e12c5a865410582e5c87816f6c3ac, entries=150, sequenceid=395, filesize=12.0 K 2024-11-15T22:12:19,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 2a24e00e990dba09c516fd169bc5958d in 997ms, sequenceid=395, compaction requested=true 2024-11-15T22:12:19,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:19,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:12:19,688 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:19,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:19,689 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:12:19,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:12:19,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:19,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:12:19,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:19,690 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:19,690 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/A is initiating minor compaction (all files) 2024-11-15T22:12:19,690 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/A in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:19,690 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/0be2dc7388294e4791b7f667598ff367, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/efce8d996e45455c8d4b8f1f6c022c96, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/1ffee492a4da48ffaa11d2a5160269c9] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=92.3 K 2024-11-15T22:12:19,690 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:19,690 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/0be2dc7388294e4791b7f667598ff367, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/efce8d996e45455c8d4b8f1f6c022c96, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/1ffee492a4da48ffaa11d2a5160269c9] 2024-11-15T22:12:19,691 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0be2dc7388294e4791b7f667598ff367, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731708736504 2024-11-15T22:12:19,692 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:12:19,692 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/B is initiating minor compaction (all files) 2024-11-15T22:12:19,692 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/B in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:19,693 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/6cdc951b02924907b846fb4cbe3a0bce, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/ffdf0ec9832a4182b85a1ea4c9978ad9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/9885ad4b9fb34fd898497bbb78e791f2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/aa28a6ed19ba4537b1517c34cdc56917] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=48.8 K 2024-11-15T22:12:19,693 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting efce8d996e45455c8d4b8f1f6c022c96, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731708736705 2024-11-15T22:12:19,693 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cdc951b02924907b846fb4cbe3a0bce, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731708735905 2024-11-15T22:12:19,694 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting ffdf0ec9832a4182b85a1ea4c9978ad9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731708736504 2024-11-15T22:12:19,694 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ffee492a4da48ffaa11d2a5160269c9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731708738008 2024-11-15T22:12:19,694 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 9885ad4b9fb34fd898497bbb78e791f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731708736705 2024-11-15T22:12:19,695 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting aa28a6ed19ba4537b1517c34cdc56917, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731708738008 2024-11-15T22:12:19,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T22:12:19,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:19,717 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-15T22:12:19,738 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:19,744 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#B#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:19,745 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/0890221813c54098bee2712da4cbd783 is 50, key is test_row_0/B:col10/1731708738690/Put/seqid=0 2024-11-15T22:12:19,746 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411158462e5b92ab24605954c7d213f74ab5c_2a24e00e990dba09c516fd169bc5958d store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:19,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115326bdd3e40974894a1e8464fac9fa1b7_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708738787/Put/seqid=0 2024-11-15T22:12:19,748 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411158462e5b92ab24605954c7d213f74ab5c_2a24e00e990dba09c516fd169bc5958d, store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:19,748 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158462e5b92ab24605954c7d213f74ab5c_2a24e00e990dba09c516fd169bc5958d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:19,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742026_1202 (size=12454) 2024-11-15T22:12:19,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742027_1203 (size=4469) 2024-11-15T22:12:19,781 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#A#compaction#159 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:19,782 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/92e261a3642b41aab87da34e1c2ecfb8 is 175, key is test_row_0/A:col10/1731708738690/Put/seqid=0 2024-11-15T22:12:19,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742025_1201 (size=13187) 2024-11-15T22:12:19,791 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/0890221813c54098bee2712da4cbd783 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/0890221813c54098bee2712da4cbd783 2024-11-15T22:12:19,798 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/B of 2a24e00e990dba09c516fd169bc5958d into 0890221813c54098bee2712da4cbd783(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:19,798 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:19,798 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/B, priority=12, startTime=1731708739689; duration=0sec 2024-11-15T22:12:19,799 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:19,799 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:B 2024-11-15T22:12:19,799 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:12:19,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742028_1204 (size=32141) 2024-11-15T22:12:19,801 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:12:19,801 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/C is initiating minor compaction (all files) 2024-11-15T22:12:19,802 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/C in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:19,802 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/37b58779b06e45ebbf2b6809e5a4e028, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/3120e16763404c2bb51f038f1a37d435, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/48f9b70d1b94480fb0a3fc3ac008a95c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/2f3e12c5a865410582e5c87816f6c3ac] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=48.8 K 2024-11-15T22:12:19,803 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 37b58779b06e45ebbf2b6809e5a4e028, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731708735905 2024-11-15T22:12:19,803 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 3120e16763404c2bb51f038f1a37d435, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731708736504 2024-11-15T22:12:19,805 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 48f9b70d1b94480fb0a3fc3ac008a95c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731708736705 2024-11-15T22:12:19,806 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f3e12c5a865410582e5c87816f6c3ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731708738008 2024-11-15T22:12:19,811 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/92e261a3642b41aab87da34e1c2ecfb8 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/92e261a3642b41aab87da34e1c2ecfb8 2024-11-15T22:12:19,822 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/A of 2a24e00e990dba09c516fd169bc5958d into 92e261a3642b41aab87da34e1c2ecfb8(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:19,823 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:19,823 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/A, priority=13, startTime=1731708739688; duration=0sec 2024-11-15T22:12:19,823 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:19,823 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:A 2024-11-15T22:12:19,826 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#C#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:19,827 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/9f3e17b84cf54e339698dd0c04ce6d9c is 50, key is test_row_0/C:col10/1731708738690/Put/seqid=0 2024-11-15T22:12:19,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742029_1205 (size=13187) 2024-11-15T22:12:19,845 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/9f3e17b84cf54e339698dd0c04ce6d9c as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/9f3e17b84cf54e339698dd0c04ce6d9c 2024-11-15T22:12:19,853 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/C of 2a24e00e990dba09c516fd169bc5958d into 9f3e17b84cf54e339698dd0c04ce6d9c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:19,853 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:19,853 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/C, priority=12, startTime=1731708739690; duration=0sec 2024-11-15T22:12:19,854 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:19,854 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:C 2024-11-15T22:12:19,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T22:12:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:19,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:20,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:20,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708800040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:20,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:20,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:20,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:20,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40758 deadline: 1731708800091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:20,095 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_1' in 2a24e00e990dba09c516fd169bc5958d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:20,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:20,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40756 deadline: 1731708800117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:20,120 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_2' in 2a24e00e990dba09c516fd169bc5958d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:20,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:20,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:20,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1731708800126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:20,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:20,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40806 deadline: 1731708800126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:20,131 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_2' in 2a24e00e990dba09c516fd169bc5958d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:20,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:20,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:20,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:20,132 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_0' in 2a24e00e990dba09c516fd169bc5958d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:20,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:20,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:20,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:20,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708800157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:20,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:20,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:20,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:20,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:20,181 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115326bdd3e40974894a1e8464fac9fa1b7_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115326bdd3e40974894a1e8464fac9fa1b7_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:20,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/7cf8bfc458a24358af200f47f9988a99, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:20,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/7cf8bfc458a24358af200f47f9988a99 is 175, key is test_row_0/A:col10/1731708738787/Put/seqid=0 2024-11-15T22:12:20,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742030_1206 (size=31255) 2024-11-15T22:12:20,206 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=413, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/7cf8bfc458a24358af200f47f9988a99 2024-11-15T22:12:20,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/5dbb639bea2542f9be98f549fb0a2688 is 50, key is test_row_0/B:col10/1731708738787/Put/seqid=0 2024-11-15T22:12:20,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742031_1207 (size=12301) 2024-11-15T22:12:20,271 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/5dbb639bea2542f9be98f549fb0a2688 2024-11-15T22:12:20,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/1983313dae084aa2b4193362644b0003 is 50, key is test_row_0/C:col10/1731708738787/Put/seqid=0 2024-11-15T22:12:20,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742032_1208 (size=12301) 2024-11-15T22:12:20,307 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/1983313dae084aa2b4193362644b0003 2024-11-15T22:12:20,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/7cf8bfc458a24358af200f47f9988a99 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/7cf8bfc458a24358af200f47f9988a99 2024-11-15T22:12:20,324 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/7cf8bfc458a24358af200f47f9988a99, entries=150, sequenceid=413, filesize=30.5 K 2024-11-15T22:12:20,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/5dbb639bea2542f9be98f549fb0a2688 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/5dbb639bea2542f9be98f549fb0a2688 2024-11-15T22:12:20,330 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/5dbb639bea2542f9be98f549fb0a2688, entries=150, sequenceid=413, filesize=12.0 K 2024-11-15T22:12:20,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/1983313dae084aa2b4193362644b0003 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/1983313dae084aa2b4193362644b0003 2024-11-15T22:12:20,342 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/1983313dae084aa2b4193362644b0003, entries=150, sequenceid=413, filesize=12.0 K 2024-11-15T22:12:20,344 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 2a24e00e990dba09c516fd169bc5958d in 626ms, sequenceid=413, compaction requested=false 2024-11-15T22:12:20,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:20,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:20,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-15T22:12:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-15T22:12:20,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-15T22:12:20,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5780 sec 2024-11-15T22:12:20,373 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:20,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T22:12:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:20,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.6000 sec 2024-11-15T22:12:20,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111552d471c978f84380bd2a98d5d4f5d186_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708740037/Put/seqid=0 2024-11-15T22:12:20,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742033_1209 (size=12454) 2024-11-15T22:12:20,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708800486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:20,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708800597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:20,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:20,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708800816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:20,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:20,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:20,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:20,844 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:20,858 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111552d471c978f84380bd2a98d5d4f5d186_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111552d471c978f84380bd2a98d5d4f5d186_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:20,862 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/b36c78d0f4fa4d6c9f7a5ff21c0c1249, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:20,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/b36c78d0f4fa4d6c9f7a5ff21c0c1249 is 175, key is test_row_0/A:col10/1731708740037/Put/seqid=0 2024-11-15T22:12:20,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742034_1210 (size=31255) 2024-11-15T22:12:20,872 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=435, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/b36c78d0f4fa4d6c9f7a5ff21c0c1249 2024-11-15T22:12:20,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/b14d8f25c4a14bd789be64e5390babd5 is 50, key is test_row_0/B:col10/1731708740037/Put/seqid=0 2024-11-15T22:12:20,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742035_1211 (size=12301) 2024-11-15T22:12:20,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/b14d8f25c4a14bd789be64e5390babd5 2024-11-15T22:12:20,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/1253f18d54f84749accde95700a78b91 is 50, key is test_row_0/C:col10/1731708740037/Put/seqid=0 2024-11-15T22:12:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T22:12:20,934 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:20,938 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:12:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-15T22:12:20,943 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:12:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T22:12:20,943 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:12:20,943 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:12:20,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742036_1212 (size=12301) 2024-11-15T22:12:20,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/1253f18d54f84749accde95700a78b91 2024-11-15T22:12:20,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/b36c78d0f4fa4d6c9f7a5ff21c0c1249 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/b36c78d0f4fa4d6c9f7a5ff21c0c1249 2024-11-15T22:12:20,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/b36c78d0f4fa4d6c9f7a5ff21c0c1249, entries=150, sequenceid=435, filesize=30.5 K 2024-11-15T22:12:20,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/b14d8f25c4a14bd789be64e5390babd5 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/b14d8f25c4a14bd789be64e5390babd5 2024-11-15T22:12:20,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/b14d8f25c4a14bd789be64e5390babd5, entries=150, sequenceid=435, filesize=12.0 K 2024-11-15T22:12:20,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/1253f18d54f84749accde95700a78b91 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/1253f18d54f84749accde95700a78b91 2024-11-15T22:12:21,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/1253f18d54f84749accde95700a78b91, entries=150, sequenceid=435, filesize=12.0 K 2024-11-15T22:12:21,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 2a24e00e990dba09c516fd169bc5958d in 635ms, sequenceid=435, compaction requested=true 2024-11-15T22:12:21,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:21,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:12:21,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:21,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:12:21,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:12:21,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:12:21,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T22:12:21,008 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:21,008 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:21,010 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:21,010 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/C is initiating minor compaction (all files) 2024-11-15T22:12:21,011 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/C in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:21,011 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/9f3e17b84cf54e339698dd0c04ce6d9c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/1983313dae084aa2b4193362644b0003, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/1253f18d54f84749accde95700a78b91] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=36.9 K 2024-11-15T22:12:21,011 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:21,011 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/A is initiating minor compaction (all files) 2024-11-15T22:12:21,011 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/A in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:21,011 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/92e261a3642b41aab87da34e1c2ecfb8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/7cf8bfc458a24358af200f47f9988a99, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/b36c78d0f4fa4d6c9f7a5ff21c0c1249] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=92.4 K 2024-11-15T22:12:21,011 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:21,011 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/92e261a3642b41aab87da34e1c2ecfb8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/7cf8bfc458a24358af200f47f9988a99, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/b36c78d0f4fa4d6c9f7a5ff21c0c1249] 2024-11-15T22:12:21,012 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f3e17b84cf54e339698dd0c04ce6d9c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731708738008 2024-11-15T22:12:21,012 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92e261a3642b41aab87da34e1c2ecfb8, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731708738008 2024-11-15T22:12:21,013 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cf8bfc458a24358af200f47f9988a99, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731708738748 2024-11-15T22:12:21,013 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 1983313dae084aa2b4193362644b0003, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731708738748 2024-11-15T22:12:21,014 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting b36c78d0f4fa4d6c9f7a5ff21c0c1249, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731708740012 2024-11-15T22:12:21,014 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 1253f18d54f84749accde95700a78b91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731708740012 2024-11-15T22:12:21,036 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#C#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:21,036 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/f13eb1113eeb47b2a76f6a29ba305bbd is 50, key is test_row_0/C:col10/1731708740037/Put/seqid=0 2024-11-15T22:12:21,038 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:21,048 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115ab5dc75e90e148c4979c72217f3e7dda_2a24e00e990dba09c516fd169bc5958d store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T22:12:21,050 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115ab5dc75e90e148c4979c72217f3e7dda_2a24e00e990dba09c516fd169bc5958d, store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:21,050 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ab5dc75e90e148c4979c72217f3e7dda_2a24e00e990dba09c516fd169bc5958d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:21,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742037_1213 (size=13289) 2024-11-15T22:12:21,078 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/f13eb1113eeb47b2a76f6a29ba305bbd as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/f13eb1113eeb47b2a76f6a29ba305bbd 2024-11-15T22:12:21,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742038_1214 (size=4469) 2024-11-15T22:12:21,087 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#A#compaction#169 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:21,088 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/527d243601804e7cae8dade9e24f02d7 is 175, key is test_row_0/A:col10/1731708740037/Put/seqid=0 2024-11-15T22:12:21,091 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/C of 2a24e00e990dba09c516fd169bc5958d into f13eb1113eeb47b2a76f6a29ba305bbd(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:21,091 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:21,091 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/C, priority=13, startTime=1731708741008; duration=0sec 2024-11-15T22:12:21,092 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:21,092 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:C 2024-11-15T22:12:21,092 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:21,094 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:21,095 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/B is initiating minor compaction (all files) 2024-11-15T22:12:21,095 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/B in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:21,095 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/0890221813c54098bee2712da4cbd783, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/5dbb639bea2542f9be98f549fb0a2688, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/b14d8f25c4a14bd789be64e5390babd5] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=36.9 K 2024-11-15T22:12:21,097 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 0890221813c54098bee2712da4cbd783, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731708738008 2024-11-15T22:12:21,098 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dbb639bea2542f9be98f549fb0a2688, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731708738748 2024-11-15T22:12:21,098 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting b14d8f25c4a14bd789be64e5390babd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731708740012 2024-11-15T22:12:21,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-15T22:12:21,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742039_1215 (size=32243) 2024-11-15T22:12:21,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:21,102 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T22:12:21,116 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/527d243601804e7cae8dade9e24f02d7 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/527d243601804e7cae8dade9e24f02d7 2024-11-15T22:12:21,124 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/A of 2a24e00e990dba09c516fd169bc5958d into 527d243601804e7cae8dade9e24f02d7(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:21,124 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:21,124 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/A, priority=13, startTime=1731708741008; duration=0sec 2024-11-15T22:12:21,124 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:21,125 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:A 2024-11-15T22:12:21,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. as already flushing 2024-11-15T22:12:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:21,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115aa33a2da54364ab0850c3b6fd9e59863_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708740406/Put/seqid=0 2024-11-15T22:12:21,140 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#B#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:21,140 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/e7c80934ff254f518850ecaff55a4b3d is 50, key is test_row_0/B:col10/1731708740037/Put/seqid=0 2024-11-15T22:12:21,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742041_1217 (size=13289) 2024-11-15T22:12:21,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742040_1216 (size=12454) 2024-11-15T22:12:21,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:21,166 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115aa33a2da54364ab0850c3b6fd9e59863_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115aa33a2da54364ab0850c3b6fd9e59863_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:21,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/33cce915441d4191aac05b97eaeafa7f, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:21,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/33cce915441d4191aac05b97eaeafa7f is 175, key is test_row_0/A:col10/1731708740406/Put/seqid=0 2024-11-15T22:12:21,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742042_1218 (size=31255) 2024-11-15T22:12:21,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:21,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708801204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:21,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:21,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:21,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:21,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T22:12:21,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:21,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708801316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:21,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:21,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:21,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:21,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708801526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:21,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:21,557 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/e7c80934ff254f518850ecaff55a4b3d as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/e7c80934ff254f518850ecaff55a4b3d 2024-11-15T22:12:21,562 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/B of 2a24e00e990dba09c516fd169bc5958d into e7c80934ff254f518850ecaff55a4b3d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:21,563 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:21,563 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/B, priority=13, startTime=1731708741008; duration=0sec 2024-11-15T22:12:21,563 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:21,563 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:B 2024-11-15T22:12:21,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T22:12:21,570 INFO [Thread-534 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-534. 2024-11-15T22:12:21,571 INFO [Thread-530 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-530. 2024-11-15T22:12:21,571 DEBUG [Thread-534 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:12:21,571 DEBUG [Thread-534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:21,571 DEBUG [Thread-530 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:12:21,571 DEBUG [Thread-530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:21,571 DEBUG [Thread-534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:21,571 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:12:21,571 INFO [Thread-528 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-528. 2024-11-15T22:12:21,571 DEBUG [Thread-528 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:12:21,571 DEBUG [Thread-528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:21,572 DEBUG [Thread-528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:21,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:12:21,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:12:21,572 DEBUG [Thread-530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:21,580 INFO [Thread-532 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-532. 2024-11-15T22:12:21,580 DEBUG [Thread-532 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:12:21,580 DEBUG [Thread-532 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:21,580 DEBUG [Thread-532 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:21,580 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:12:21,580 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=453, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/33cce915441d4191aac05b97eaeafa7f 2024-11-15T22:12:21,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/a184848547b44973a8569193c107b70a is 50, key is test_row_0/B:col10/1731708740406/Put/seqid=0 2024-11-15T22:12:21,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742043_1219 (size=12301) 2024-11-15T22:12:21,595 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/a184848547b44973a8569193c107b70a 2024-11-15T22:12:21,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/ccc00b1a79b345aa831ce0cbcaff21e0 is 50, key is test_row_0/C:col10/1731708740406/Put/seqid=0 2024-11-15T22:12:21,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742044_1220 (size=12301) 2024-11-15T22:12:21,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1731708801836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:21,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:12:22,012 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/ccc00b1a79b345aa831ce0cbcaff21e0 2024-11-15T22:12:22,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/33cce915441d4191aac05b97eaeafa7f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/33cce915441d4191aac05b97eaeafa7f 2024-11-15T22:12:22,023 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/33cce915441d4191aac05b97eaeafa7f, entries=150, sequenceid=453, filesize=30.5 K 2024-11-15T22:12:22,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/a184848547b44973a8569193c107b70a as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/a184848547b44973a8569193c107b70a 2024-11-15T22:12:22,029 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/a184848547b44973a8569193c107b70a, entries=150, sequenceid=453, filesize=12.0 K 2024-11-15T22:12:22,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/ccc00b1a79b345aa831ce0cbcaff21e0 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/ccc00b1a79b345aa831ce0cbcaff21e0 2024-11-15T22:12:22,034 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/ccc00b1a79b345aa831ce0cbcaff21e0, entries=150, sequenceid=453, filesize=12.0 K 2024-11-15T22:12:22,035 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 2a24e00e990dba09c516fd169bc5958d in 933ms, sequenceid=453, compaction requested=false 2024-11-15T22:12:22,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:22,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:22,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-15T22:12:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-15T22:12:22,039 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-15T22:12:22,039 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0940 sec 2024-11-15T22:12:22,042 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.1020 sec 2024-11-15T22:12:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T22:12:22,077 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:22,347 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:22,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-15T22:12:22,348 INFO [Thread-523 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-523. 2024-11-15T22:12:22,348 DEBUG [Thread-523 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:12:22,348 DEBUG [Thread-523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:22,348 DEBUG [Thread-523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:22,348 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:12:22,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a181402c43524c84b086fb87b87bcaaf_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708741198/Put/seqid=0 2024-11-15T22:12:22,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742045_1221 (size=12454) 2024-11-15T22:12:22,759 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:22,763 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a181402c43524c84b086fb87b87bcaaf_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a181402c43524c84b086fb87b87bcaaf_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:22,764 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/6e08837ae2ea471584a5259926ad1dcc, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:22,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/6e08837ae2ea471584a5259926ad1dcc is 175, key is test_row_0/A:col10/1731708741198/Put/seqid=0 2024-11-15T22:12:22,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742046_1222 (size=31255) 2024-11-15T22:12:23,170 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=475, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/6e08837ae2ea471584a5259926ad1dcc 2024-11-15T22:12:23,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/5d029b2320fc475d986b264f443f1ab9 is 50, key is test_row_0/B:col10/1731708741198/Put/seqid=0 2024-11-15T22:12:23,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742047_1223 (size=12301) 2024-11-15T22:12:23,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/5d029b2320fc475d986b264f443f1ab9 2024-11-15T22:12:23,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/cb98a1bcb69d4715a70081cd15be8dfc is 50, key is test_row_0/C:col10/1731708741198/Put/seqid=0 2024-11-15T22:12:23,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742048_1224 (size=12301) 2024-11-15T22:12:23,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/cb98a1bcb69d4715a70081cd15be8dfc 2024-11-15T22:12:24,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/6e08837ae2ea471584a5259926ad1dcc as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/6e08837ae2ea471584a5259926ad1dcc 2024-11-15T22:12:24,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/6e08837ae2ea471584a5259926ad1dcc, entries=150, sequenceid=475, filesize=30.5 K 2024-11-15T22:12:24,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/5d029b2320fc475d986b264f443f1ab9 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/5d029b2320fc475d986b264f443f1ab9 2024-11-15T22:12:24,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/5d029b2320fc475d986b264f443f1ab9, entries=150, sequenceid=475, filesize=12.0 K 2024-11-15T22:12:24,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/cb98a1bcb69d4715a70081cd15be8dfc as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/cb98a1bcb69d4715a70081cd15be8dfc 2024-11-15T22:12:24,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/cb98a1bcb69d4715a70081cd15be8dfc, entries=150, sequenceid=475, filesize=12.0 K 2024-11-15T22:12:24,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 2a24e00e990dba09c516fd169bc5958d in 1667ms, sequenceid=475, compaction requested=true 2024-11-15T22:12:24,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:24,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:12:24,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:24,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:12:24,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:24,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a24e00e990dba09c516fd169bc5958d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:12:24,014 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:24,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:24,014 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:24,015 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:24,015 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94753 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:24,015 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/B is initiating minor compaction (all files) 2024-11-15T22:12:24,015 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a24e00e990dba09c516fd169bc5958d/A is initiating minor compaction (all files) 2024-11-15T22:12:24,015 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/B in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:24,015 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a24e00e990dba09c516fd169bc5958d/A in TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:24,015 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/e7c80934ff254f518850ecaff55a4b3d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/a184848547b44973a8569193c107b70a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/5d029b2320fc475d986b264f443f1ab9] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=37.0 K 2024-11-15T22:12:24,015 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/527d243601804e7cae8dade9e24f02d7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/33cce915441d4191aac05b97eaeafa7f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/6e08837ae2ea471584a5259926ad1dcc] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp, totalSize=92.5 K 2024-11-15T22:12:24,015 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:24,015 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/527d243601804e7cae8dade9e24f02d7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/33cce915441d4191aac05b97eaeafa7f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/6e08837ae2ea471584a5259926ad1dcc] 2024-11-15T22:12:24,016 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting e7c80934ff254f518850ecaff55a4b3d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731708740012 2024-11-15T22:12:24,016 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 527d243601804e7cae8dade9e24f02d7, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731708740012 2024-11-15T22:12:24,016 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting a184848547b44973a8569193c107b70a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731708740406 2024-11-15T22:12:24,016 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33cce915441d4191aac05b97eaeafa7f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731708740406 2024-11-15T22:12:24,016 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d029b2320fc475d986b264f443f1ab9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731708741193 2024-11-15T22:12:24,017 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e08837ae2ea471584a5259926ad1dcc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731708741193 2024-11-15T22:12:24,025 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#B#compaction#177 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:24,025 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/71409d61733843e38f44f4fb031058b4 is 50, key is test_row_0/B:col10/1731708741198/Put/seqid=0 2024-11-15T22:12:24,028 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:24,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742049_1225 (size=13391) 2024-11-15T22:12:24,034 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115599e8f85b5d742b78953388d9b0b8498_2a24e00e990dba09c516fd169bc5958d store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:24,066 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115599e8f85b5d742b78953388d9b0b8498_2a24e00e990dba09c516fd169bc5958d, store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:24,066 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115599e8f85b5d742b78953388d9b0b8498_2a24e00e990dba09c516fd169bc5958d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:24,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742050_1226 (size=4469) 2024-11-15T22:12:24,111 INFO [Thread-521 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-521. 2024-11-15T22:12:24,111 DEBUG [Thread-521 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:12:24,111 DEBUG [Thread-521 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:24,111 DEBUG [Thread-521 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:24,111 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:12:24,139 INFO [Thread-517 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-517. 2024-11-15T22:12:24,139 DEBUG [Thread-517 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:12:24,139 DEBUG [Thread-517 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:24,140 DEBUG [Thread-517 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:24,140 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:12:24,173 INFO [Thread-525 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-525. 2024-11-15T22:12:24,173 DEBUG [Thread-525 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:12:24,173 DEBUG [Thread-525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:24,173 DEBUG [Thread-525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:24,174 INFO [Thread-519 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-519. 2024-11-15T22:12:24,174 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:12:24,174 DEBUG [Thread-519 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:12:24,174 DEBUG [Thread-519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:24,174 DEBUG [Thread-519 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:24,174 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:12:24,174 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T22:12:24,174 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-15T22:12:24,174 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-15T22:12:24,174 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-15T22:12:24,174 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 160 2024-11-15T22:12:24,174 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-15T22:12:24,174 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T22:12:24,174 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2493 2024-11-15T22:12:24,174 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2422 2024-11-15T22:12:24,174 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T22:12:24,174 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1127 2024-11-15T22:12:24,174 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3381 rows 2024-11-15T22:12:24,174 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1119 2024-11-15T22:12:24,174 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3354 rows 2024-11-15T22:12:24,175 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T22:12:24,175 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:12:24,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:24,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:24,175 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:12:24,178 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T22:12:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T22:12:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T22:12:24,182 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708744182"}]},"ts":"1731708744182"} 2024-11-15T22:12:24,184 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T22:12:24,184 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T22:12:24,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T22:12:24,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a24e00e990dba09c516fd169bc5958d, UNASSIGN}] 2024-11-15T22:12:24,188 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a24e00e990dba09c516fd169bc5958d, UNASSIGN 2024-11-15T22:12:24,189 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=2a24e00e990dba09c516fd169bc5958d, regionState=CLOSING, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:24,191 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a24e00e990dba09c516fd169bc5958d, UNASSIGN because future has completed 2024-11-15T22:12:24,191 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T22:12:24,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516}] 2024-11-15T22:12:24,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T22:12:24,344 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:24,344 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T22:12:24,344 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 2a24e00e990dba09c516fd169bc5958d, disabling compactions & flushes 2024-11-15T22:12:24,344 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:24,437 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/71409d61733843e38f44f4fb031058b4 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/71409d61733843e38f44f4fb031058b4 2024-11-15T22:12:24,441 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/B of 2a24e00e990dba09c516fd169bc5958d into 71409d61733843e38f44f4fb031058b4(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:24,442 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:24,442 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/B, priority=13, startTime=1731708744014; duration=0sec 2024-11-15T22:12:24,442 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:24,442 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:B 2024-11-15T22:12:24,442 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. because compaction request was cancelled 2024-11-15T22:12:24,442 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:C 2024-11-15T22:12:24,473 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a24e00e990dba09c516fd169bc5958d#A#compaction#178 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:24,473 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/57ffce1d29ef4b5994efb3b258439324 is 175, key is test_row_0/A:col10/1731708741198/Put/seqid=0 2024-11-15T22:12:24,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742051_1227 (size=32345) 2024-11-15T22:12:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T22:12:24,574 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T22:12:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T22:12:24,893 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/57ffce1d29ef4b5994efb3b258439324 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/57ffce1d29ef4b5994efb3b258439324 2024-11-15T22:12:24,910 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a24e00e990dba09c516fd169bc5958d/A of 2a24e00e990dba09c516fd169bc5958d into 57ffce1d29ef4b5994efb3b258439324(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:24,910 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a24e00e990dba09c516fd169bc5958d: 2024-11-15T22:12:24,910 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., storeName=2a24e00e990dba09c516fd169bc5958d/A, priority=13, startTime=1731708744014; duration=0sec 2024-11-15T22:12:24,910 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:24,910 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:24,910 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a24e00e990dba09c516fd169bc5958d:A 2024-11-15T22:12:24,910 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:24,910 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. after waiting 0 ms 2024-11-15T22:12:24,910 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:24,910 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 2a24e00e990dba09c516fd169bc5958d 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-15T22:12:24,925 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156d6525df00ae43cf9fad748a0c4bc608_2a24e00e990dba09c516fd169bc5958d is 50, key is test_row_0/A:col10/1731708744167/Put/seqid=0 2024-11-15T22:12:24,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742052_1228 (size=12454) 2024-11-15T22:12:25,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T22:12:25,339 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:25,343 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156d6525df00ae43cf9fad748a0c4bc608_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156d6525df00ae43cf9fad748a0c4bc608_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:25,344 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/2fd01205279143b885aab640f3e35bb2, store: [table=TestAcidGuarantees family=A region=2a24e00e990dba09c516fd169bc5958d] 2024-11-15T22:12:25,345 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/2fd01205279143b885aab640f3e35bb2 is 175, key is test_row_0/A:col10/1731708744167/Put/seqid=0 2024-11-15T22:12:25,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742053_1229 (size=31255) 2024-11-15T22:12:25,749 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=484, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/2fd01205279143b885aab640f3e35bb2 2024-11-15T22:12:25,758 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/293ec75bceb941a990b3aed8e607bc22 is 50, key is test_row_0/B:col10/1731708744167/Put/seqid=0 2024-11-15T22:12:25,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742054_1230 (size=12301) 2024-11-15T22:12:26,167 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/293ec75bceb941a990b3aed8e607bc22 2024-11-15T22:12:26,176 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/831adfb10d2e4296b01a22372f750435 is 50, key is test_row_0/C:col10/1731708744167/Put/seqid=0 2024-11-15T22:12:26,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742055_1231 (size=12301) 2024-11-15T22:12:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T22:12:26,581 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/831adfb10d2e4296b01a22372f750435 2024-11-15T22:12:26,586 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/A/2fd01205279143b885aab640f3e35bb2 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/2fd01205279143b885aab640f3e35bb2 2024-11-15T22:12:26,590 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/2fd01205279143b885aab640f3e35bb2, entries=150, sequenceid=484, filesize=30.5 K 2024-11-15T22:12:26,591 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/B/293ec75bceb941a990b3aed8e607bc22 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/293ec75bceb941a990b3aed8e607bc22 2024-11-15T22:12:26,596 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/293ec75bceb941a990b3aed8e607bc22, entries=150, sequenceid=484, filesize=12.0 K 2024-11-15T22:12:26,597 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/.tmp/C/831adfb10d2e4296b01a22372f750435 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/831adfb10d2e4296b01a22372f750435 2024-11-15T22:12:26,600 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/831adfb10d2e4296b01a22372f750435, entries=150, sequenceid=484, filesize=12.0 K 2024-11-15T22:12:26,601 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 2a24e00e990dba09c516fd169bc5958d in 1691ms, sequenceid=484, compaction requested=true 2024-11-15T22:12:26,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/0b2bf20d3f144011a2263a9cfb34fc22, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/3de55b29846b4c90862b0f7f18593cca, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/77f6eef5fd6b483a8bb09b69a4d1b3fb, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/aa9cd11997d0410f8aefd7bbac219861, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/3bd5e16bf7f34abeb5f537d4bab391de, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/9ca4b834879e4cd1bf2c678ad2c63217, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/14e12ed7675d460ab15ae5aee011892b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/670e59c3dbe74a04af50d7521e4e0a47, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/afe19724cccd4db99c06ac0e39c36eab, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/980ad2107b5442c8850a5571dd7e6111, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/5416bbc06db24199901d549c88586592, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/454c4e659c8a40918094ececd6472a01, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/cd9dd4069867407599781411ce6697c7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/d27c63f5e1984e9f93f9e293ec3e0a55, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/612ba0cb7f394cb4a7c28d2f5148b177, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/12b2608f10c248e3aafe67324e05d0ef, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/46030bf4ce074b9b8dc2602f668dd3c6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/b2838c5d32d64eec8b36c71070530809, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/d23c70e7f0c04580aa7270ef3eb804e6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/12d9edffd35f4e2695fc45b5a373e025, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/d9e99cf6b3b84c58b0750b93d10e9c3b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/a9bbbf08c3d04cf7adf43dabf233ee0d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/191a0139bf1b435884b0e060b2db73bb, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/0be2dc7388294e4791b7f667598ff367, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/efce8d996e45455c8d4b8f1f6c022c96, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/92e261a3642b41aab87da34e1c2ecfb8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/1ffee492a4da48ffaa11d2a5160269c9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/7cf8bfc458a24358af200f47f9988a99, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/527d243601804e7cae8dade9e24f02d7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/b36c78d0f4fa4d6c9f7a5ff21c0c1249, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/33cce915441d4191aac05b97eaeafa7f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/6e08837ae2ea471584a5259926ad1dcc] to archive 2024-11-15T22:12:26,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:12:26,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/0b2bf20d3f144011a2263a9cfb34fc22 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/0b2bf20d3f144011a2263a9cfb34fc22 2024-11-15T22:12:26,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/3de55b29846b4c90862b0f7f18593cca to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/3de55b29846b4c90862b0f7f18593cca 2024-11-15T22:12:26,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/77f6eef5fd6b483a8bb09b69a4d1b3fb to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/77f6eef5fd6b483a8bb09b69a4d1b3fb 2024-11-15T22:12:26,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/aa9cd11997d0410f8aefd7bbac219861 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/aa9cd11997d0410f8aefd7bbac219861 2024-11-15T22:12:26,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/3bd5e16bf7f34abeb5f537d4bab391de to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/3bd5e16bf7f34abeb5f537d4bab391de 2024-11-15T22:12:26,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/9ca4b834879e4cd1bf2c678ad2c63217 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/9ca4b834879e4cd1bf2c678ad2c63217 2024-11-15T22:12:26,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/14e12ed7675d460ab15ae5aee011892b to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/14e12ed7675d460ab15ae5aee011892b 2024-11-15T22:12:26,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/670e59c3dbe74a04af50d7521e4e0a47 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/670e59c3dbe74a04af50d7521e4e0a47 2024-11-15T22:12:26,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/afe19724cccd4db99c06ac0e39c36eab to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/afe19724cccd4db99c06ac0e39c36eab 2024-11-15T22:12:26,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/980ad2107b5442c8850a5571dd7e6111 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/980ad2107b5442c8850a5571dd7e6111 2024-11-15T22:12:26,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/5416bbc06db24199901d549c88586592 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/5416bbc06db24199901d549c88586592 2024-11-15T22:12:26,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/454c4e659c8a40918094ececd6472a01 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/454c4e659c8a40918094ececd6472a01 2024-11-15T22:12:26,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/cd9dd4069867407599781411ce6697c7 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/cd9dd4069867407599781411ce6697c7 2024-11-15T22:12:26,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/d27c63f5e1984e9f93f9e293ec3e0a55 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/d27c63f5e1984e9f93f9e293ec3e0a55 2024-11-15T22:12:26,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/612ba0cb7f394cb4a7c28d2f5148b177 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/612ba0cb7f394cb4a7c28d2f5148b177 2024-11-15T22:12:26,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/12b2608f10c248e3aafe67324e05d0ef to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/12b2608f10c248e3aafe67324e05d0ef 2024-11-15T22:12:26,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/46030bf4ce074b9b8dc2602f668dd3c6 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/46030bf4ce074b9b8dc2602f668dd3c6 2024-11-15T22:12:26,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/b2838c5d32d64eec8b36c71070530809 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/b2838c5d32d64eec8b36c71070530809 2024-11-15T22:12:26,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/d23c70e7f0c04580aa7270ef3eb804e6 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/d23c70e7f0c04580aa7270ef3eb804e6 2024-11-15T22:12:26,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/12d9edffd35f4e2695fc45b5a373e025 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/12d9edffd35f4e2695fc45b5a373e025 2024-11-15T22:12:26,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/d9e99cf6b3b84c58b0750b93d10e9c3b to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/d9e99cf6b3b84c58b0750b93d10e9c3b 2024-11-15T22:12:26,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/a9bbbf08c3d04cf7adf43dabf233ee0d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/a9bbbf08c3d04cf7adf43dabf233ee0d 2024-11-15T22:12:26,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/191a0139bf1b435884b0e060b2db73bb to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/191a0139bf1b435884b0e060b2db73bb 2024-11-15T22:12:26,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/0be2dc7388294e4791b7f667598ff367 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/0be2dc7388294e4791b7f667598ff367 2024-11-15T22:12:26,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/efce8d996e45455c8d4b8f1f6c022c96 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/efce8d996e45455c8d4b8f1f6c022c96 2024-11-15T22:12:26,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/92e261a3642b41aab87da34e1c2ecfb8 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/92e261a3642b41aab87da34e1c2ecfb8 2024-11-15T22:12:26,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/1ffee492a4da48ffaa11d2a5160269c9 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/1ffee492a4da48ffaa11d2a5160269c9 2024-11-15T22:12:26,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/7cf8bfc458a24358af200f47f9988a99 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/7cf8bfc458a24358af200f47f9988a99 2024-11-15T22:12:26,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/527d243601804e7cae8dade9e24f02d7 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/527d243601804e7cae8dade9e24f02d7 2024-11-15T22:12:26,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/b36c78d0f4fa4d6c9f7a5ff21c0c1249 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/b36c78d0f4fa4d6c9f7a5ff21c0c1249 2024-11-15T22:12:26,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/33cce915441d4191aac05b97eaeafa7f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/33cce915441d4191aac05b97eaeafa7f 2024-11-15T22:12:26,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/6e08837ae2ea471584a5259926ad1dcc to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/6e08837ae2ea471584a5259926ad1dcc 2024-11-15T22:12:26,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/3617d1b73dae42fc9dccb219c0100b1e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/4163c40179b1444e82e7ccc856c2e08f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/b18568ebfc77476e95028bbb48bfda8d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/291f0eae3c494da0ac8c32cc04e7058f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/8d65933978b7460d97a379447b1a4e84, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/d29b182c11af4fb5ad4ff20efbeec23f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/c1d0ec4ac1f848d38a3eabe41032ea96, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/76760b50a9ae464b96b035973c0c458a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/6ef2612327f1447fa33035bcac432061, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/5c783ebc160d4fc2a9aa1d7c63a953cd, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/31d9f0088f574d69b500094bf7146f68, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/f4e94261dc3c4cf4b1414fd26027a376, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/9237f462bdfb4ec6886cfd683307a9c0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/aa3b5263de4b451e9ce5e351fbb336ad, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/d2e1451635a14e56944ddedfe2e89a5e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/da418c39dd044183bbf2fbebf7747994, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/aeeaca18fbee492aad55e40ef9798ed3, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/ed3ea6388bde43fe9b6c1c3f56ff14c5, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/b5d978be02d4495081bd69c94fdb144a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/515fe90653ff4693a518b372fec0ed77, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/67c871c841794d8da3549730bc4e7cd8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/6cdc951b02924907b846fb4cbe3a0bce, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/361386c5e9e244499f1cacb1d7b1aea9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/ffdf0ec9832a4182b85a1ea4c9978ad9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/9885ad4b9fb34fd898497bbb78e791f2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/0890221813c54098bee2712da4cbd783, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/aa28a6ed19ba4537b1517c34cdc56917, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/5dbb639bea2542f9be98f549fb0a2688, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/e7c80934ff254f518850ecaff55a4b3d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/b14d8f25c4a14bd789be64e5390babd5, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/a184848547b44973a8569193c107b70a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/5d029b2320fc475d986b264f443f1ab9] to archive 2024-11-15T22:12:26,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:12:26,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/3617d1b73dae42fc9dccb219c0100b1e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/3617d1b73dae42fc9dccb219c0100b1e 2024-11-15T22:12:26,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/4163c40179b1444e82e7ccc856c2e08f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/4163c40179b1444e82e7ccc856c2e08f 2024-11-15T22:12:26,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/b18568ebfc77476e95028bbb48bfda8d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/b18568ebfc77476e95028bbb48bfda8d 2024-11-15T22:12:26,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/291f0eae3c494da0ac8c32cc04e7058f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/291f0eae3c494da0ac8c32cc04e7058f 2024-11-15T22:12:26,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/8d65933978b7460d97a379447b1a4e84 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/8d65933978b7460d97a379447b1a4e84 2024-11-15T22:12:26,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/d29b182c11af4fb5ad4ff20efbeec23f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/d29b182c11af4fb5ad4ff20efbeec23f 2024-11-15T22:12:26,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/c1d0ec4ac1f848d38a3eabe41032ea96 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/c1d0ec4ac1f848d38a3eabe41032ea96 2024-11-15T22:12:26,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/76760b50a9ae464b96b035973c0c458a to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/76760b50a9ae464b96b035973c0c458a 2024-11-15T22:12:26,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/6ef2612327f1447fa33035bcac432061 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/6ef2612327f1447fa33035bcac432061 2024-11-15T22:12:26,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/5c783ebc160d4fc2a9aa1d7c63a953cd to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/5c783ebc160d4fc2a9aa1d7c63a953cd 2024-11-15T22:12:26,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/31d9f0088f574d69b500094bf7146f68 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/31d9f0088f574d69b500094bf7146f68 2024-11-15T22:12:26,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/f4e94261dc3c4cf4b1414fd26027a376 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/f4e94261dc3c4cf4b1414fd26027a376 2024-11-15T22:12:26,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/9237f462bdfb4ec6886cfd683307a9c0 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/9237f462bdfb4ec6886cfd683307a9c0 2024-11-15T22:12:26,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/aa3b5263de4b451e9ce5e351fbb336ad to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/aa3b5263de4b451e9ce5e351fbb336ad 2024-11-15T22:12:26,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/d2e1451635a14e56944ddedfe2e89a5e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/d2e1451635a14e56944ddedfe2e89a5e 2024-11-15T22:12:26,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/da418c39dd044183bbf2fbebf7747994 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/da418c39dd044183bbf2fbebf7747994 2024-11-15T22:12:26,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/aeeaca18fbee492aad55e40ef9798ed3 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/aeeaca18fbee492aad55e40ef9798ed3 2024-11-15T22:12:26,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/ed3ea6388bde43fe9b6c1c3f56ff14c5 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/ed3ea6388bde43fe9b6c1c3f56ff14c5 2024-11-15T22:12:26,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/b5d978be02d4495081bd69c94fdb144a to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/b5d978be02d4495081bd69c94fdb144a 2024-11-15T22:12:26,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/515fe90653ff4693a518b372fec0ed77 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/515fe90653ff4693a518b372fec0ed77 2024-11-15T22:12:26,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/67c871c841794d8da3549730bc4e7cd8 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/67c871c841794d8da3549730bc4e7cd8 2024-11-15T22:12:26,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/6cdc951b02924907b846fb4cbe3a0bce to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/6cdc951b02924907b846fb4cbe3a0bce 2024-11-15T22:12:26,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/361386c5e9e244499f1cacb1d7b1aea9 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/361386c5e9e244499f1cacb1d7b1aea9 2024-11-15T22:12:26,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/ffdf0ec9832a4182b85a1ea4c9978ad9 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/ffdf0ec9832a4182b85a1ea4c9978ad9 2024-11-15T22:12:26,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/9885ad4b9fb34fd898497bbb78e791f2 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/9885ad4b9fb34fd898497bbb78e791f2 2024-11-15T22:12:26,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/0890221813c54098bee2712da4cbd783 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/0890221813c54098bee2712da4cbd783 2024-11-15T22:12:26,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/aa28a6ed19ba4537b1517c34cdc56917 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/aa28a6ed19ba4537b1517c34cdc56917 2024-11-15T22:12:26,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/5dbb639bea2542f9be98f549fb0a2688 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/5dbb639bea2542f9be98f549fb0a2688 2024-11-15T22:12:26,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/e7c80934ff254f518850ecaff55a4b3d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/e7c80934ff254f518850ecaff55a4b3d 2024-11-15T22:12:26,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/b14d8f25c4a14bd789be64e5390babd5 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/b14d8f25c4a14bd789be64e5390babd5 2024-11-15T22:12:26,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/a184848547b44973a8569193c107b70a to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/a184848547b44973a8569193c107b70a 2024-11-15T22:12:26,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/5d029b2320fc475d986b264f443f1ab9 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/5d029b2320fc475d986b264f443f1ab9 2024-11-15T22:12:26,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/f9b316d5f7404284a543cd02319c8e72, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/8a35bc740b5a47dc8bce92ccad3d63a5, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/9c5a160417b349d68c710e54619dbfbb, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/259b92e04b1e4bb7aef3f51ef2dc5464, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/a15896c2b2514407a8e837a65ac2b4bc, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/121f286c4b0d4dd7ae5edb9b11869445, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/5cdc0d4bbfc143eda93acea988ceeafd, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/32af93aa691645f6861e704b7c446c44, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/bc510524fc1e4babb15fa107eb22ffd6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/af61a656f3f64383a6a5a019bf2855e7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/4e6ab6ebf86a4f19b98ec85cc877cbf9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/e2caeaad18a94684a81f2e83c02128a1, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/1398c4e293654128ab31554715a40625, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/9930bdda810f453ea5a3bbd6dace68f1, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/f17eb6c784ca4eec97b17f4cfc7f9c9b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/293a2748ebb9424a979c3e4eb6eab9a4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/3a9f1ed7c87642818a248ac91d840a28, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/818d7c09176846989a531bf9a9169143, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/afc8f63a823141d28696a35e528e5c06, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/e88821a670a54466814df8ec96d9df9b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/f25e8d1894124ddbad8ef231e12d0f74, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/37b58779b06e45ebbf2b6809e5a4e028, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/bf6b147a3339400b87bdb9810af7decb, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/3120e16763404c2bb51f038f1a37d435, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/48f9b70d1b94480fb0a3fc3ac008a95c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/9f3e17b84cf54e339698dd0c04ce6d9c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/2f3e12c5a865410582e5c87816f6c3ac, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/1983313dae084aa2b4193362644b0003, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/1253f18d54f84749accde95700a78b91] to archive 2024-11-15T22:12:26,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:12:26,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/f9b316d5f7404284a543cd02319c8e72 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/f9b316d5f7404284a543cd02319c8e72 2024-11-15T22:12:26,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/8a35bc740b5a47dc8bce92ccad3d63a5 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/8a35bc740b5a47dc8bce92ccad3d63a5 2024-11-15T22:12:26,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/9c5a160417b349d68c710e54619dbfbb to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/9c5a160417b349d68c710e54619dbfbb 2024-11-15T22:12:26,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/259b92e04b1e4bb7aef3f51ef2dc5464 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/259b92e04b1e4bb7aef3f51ef2dc5464 2024-11-15T22:12:26,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/a15896c2b2514407a8e837a65ac2b4bc to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/a15896c2b2514407a8e837a65ac2b4bc 2024-11-15T22:12:26,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/121f286c4b0d4dd7ae5edb9b11869445 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/121f286c4b0d4dd7ae5edb9b11869445 2024-11-15T22:12:26,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/5cdc0d4bbfc143eda93acea988ceeafd to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/5cdc0d4bbfc143eda93acea988ceeafd 2024-11-15T22:12:26,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/32af93aa691645f6861e704b7c446c44 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/32af93aa691645f6861e704b7c446c44 2024-11-15T22:12:26,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/bc510524fc1e4babb15fa107eb22ffd6 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/bc510524fc1e4babb15fa107eb22ffd6 2024-11-15T22:12:26,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/af61a656f3f64383a6a5a019bf2855e7 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/af61a656f3f64383a6a5a019bf2855e7 2024-11-15T22:12:26,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/4e6ab6ebf86a4f19b98ec85cc877cbf9 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/4e6ab6ebf86a4f19b98ec85cc877cbf9 2024-11-15T22:12:26,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/e2caeaad18a94684a81f2e83c02128a1 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/e2caeaad18a94684a81f2e83c02128a1 2024-11-15T22:12:26,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/1398c4e293654128ab31554715a40625 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/1398c4e293654128ab31554715a40625 2024-11-15T22:12:26,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/9930bdda810f453ea5a3bbd6dace68f1 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/9930bdda810f453ea5a3bbd6dace68f1 2024-11-15T22:12:26,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/f17eb6c784ca4eec97b17f4cfc7f9c9b to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/f17eb6c784ca4eec97b17f4cfc7f9c9b 2024-11-15T22:12:26,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/293a2748ebb9424a979c3e4eb6eab9a4 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/293a2748ebb9424a979c3e4eb6eab9a4 2024-11-15T22:12:26,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/3a9f1ed7c87642818a248ac91d840a28 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/3a9f1ed7c87642818a248ac91d840a28 2024-11-15T22:12:26,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/818d7c09176846989a531bf9a9169143 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/818d7c09176846989a531bf9a9169143 2024-11-15T22:12:26,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/afc8f63a823141d28696a35e528e5c06 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/afc8f63a823141d28696a35e528e5c06 2024-11-15T22:12:26,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/e88821a670a54466814df8ec96d9df9b to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/e88821a670a54466814df8ec96d9df9b 2024-11-15T22:12:26,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/f25e8d1894124ddbad8ef231e12d0f74 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/f25e8d1894124ddbad8ef231e12d0f74 2024-11-15T22:12:26,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/37b58779b06e45ebbf2b6809e5a4e028 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/37b58779b06e45ebbf2b6809e5a4e028 2024-11-15T22:12:26,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/bf6b147a3339400b87bdb9810af7decb to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/bf6b147a3339400b87bdb9810af7decb 2024-11-15T22:12:26,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/3120e16763404c2bb51f038f1a37d435 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/3120e16763404c2bb51f038f1a37d435 2024-11-15T22:12:26,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/48f9b70d1b94480fb0a3fc3ac008a95c to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/48f9b70d1b94480fb0a3fc3ac008a95c 2024-11-15T22:12:26,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/9f3e17b84cf54e339698dd0c04ce6d9c to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/9f3e17b84cf54e339698dd0c04ce6d9c 2024-11-15T22:12:26,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/2f3e12c5a865410582e5c87816f6c3ac to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/2f3e12c5a865410582e5c87816f6c3ac 2024-11-15T22:12:26,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/1983313dae084aa2b4193362644b0003 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/1983313dae084aa2b4193362644b0003 2024-11-15T22:12:26,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/1253f18d54f84749accde95700a78b91 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/1253f18d54f84749accde95700a78b91 2024-11-15T22:12:26,720 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/recovered.edits/487.seqid, newMaxSeqId=487, maxSeqId=4 2024-11-15T22:12:26,721 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. 2024-11-15T22:12:26,721 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 2a24e00e990dba09c516fd169bc5958d: Waiting for close lock at 1731708744344Running coprocessor pre-close hooks at 1731708744344Disabling compacts and flushes for region at 1731708744344Disabling writes for close at 1731708744910 (+566 ms)Obtaining lock to block concurrent updates at 1731708744910Preparing flush snapshotting stores in 2a24e00e990dba09c516fd169bc5958d at 1731708744910Finished memstore snapshotting TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731708744910Flushing stores of TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d. at 1731708744911 (+1 ms)Flushing 2a24e00e990dba09c516fd169bc5958d/A: creating writer at 1731708744911Flushing mob file 2a24e00e990dba09c516fd169bc5958d/A: appending metadata at 1731708744925 (+14 ms)Flushing mob file 2a24e00e990dba09c516fd169bc5958d/A: closing flushed file at 1731708744925Flushing 2a24e00e990dba09c516fd169bc5958d/A: appending metadata at 1731708745344 (+419 ms)Flushing 2a24e00e990dba09c516fd169bc5958d/A: closing flushed file at 1731708745344Flushing 2a24e00e990dba09c516fd169bc5958d/B: creating writer at 1731708745754 (+410 ms)Flushing 2a24e00e990dba09c516fd169bc5958d/B: appending metadata at 1731708745758 (+4 ms)Flushing 2a24e00e990dba09c516fd169bc5958d/B: closing flushed file at 1731708745758Flushing 2a24e00e990dba09c516fd169bc5958d/C: creating writer at 1731708746172 (+414 ms)Flushing 2a24e00e990dba09c516fd169bc5958d/C: appending metadata at 1731708746175 (+3 ms)Flushing 2a24e00e990dba09c516fd169bc5958d/C: closing flushed file at 1731708746176 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d90dcd4: reopening flushed file at 1731708746586 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58088467: reopening flushed file at 1731708746591 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d66c576: reopening flushed file at 1731708746596 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 2a24e00e990dba09c516fd169bc5958d in 1691ms, sequenceid=484, compaction requested=true at 1731708746601 (+5 ms)Writing region close event to WAL at 1731708746716 (+115 ms)Running coprocessor post-close hooks at 1731708746721 (+5 ms)Closed at 1731708746721 2024-11-15T22:12:26,723 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:26,724 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=2a24e00e990dba09c516fd169bc5958d, regionState=CLOSED 2024-11-15T22:12:26,725 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 because future has completed 2024-11-15T22:12:26,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-15T22:12:26,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 2a24e00e990dba09c516fd169bc5958d, server=ae3311ea5ccb,37917,1731708686516 in 2.5350 sec 2024-11-15T22:12:26,729 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-15T22:12:26,729 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a24e00e990dba09c516fd169bc5958d, UNASSIGN in 2.5410 sec 2024-11-15T22:12:26,731 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-15T22:12:26,731 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5450 sec 2024-11-15T22:12:26,732 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708746732"}]},"ts":"1731708746732"} 2024-11-15T22:12:26,733 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T22:12:26,733 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T22:12:26,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5560 sec 2024-11-15T22:12:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T22:12:28,336 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:28,337 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T22:12:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:12:28,338 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:12:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T22:12:28,339 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:12:28,341 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,344 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A, FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B, FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C, FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/recovered.edits] 2024-11-15T22:12:28,347 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/2fd01205279143b885aab640f3e35bb2 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/2fd01205279143b885aab640f3e35bb2 2024-11-15T22:12:28,348 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/57ffce1d29ef4b5994efb3b258439324 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/A/57ffce1d29ef4b5994efb3b258439324 2024-11-15T22:12:28,351 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/293ec75bceb941a990b3aed8e607bc22 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/293ec75bceb941a990b3aed8e607bc22 2024-11-15T22:12:28,352 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/71409d61733843e38f44f4fb031058b4 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/B/71409d61733843e38f44f4fb031058b4 2024-11-15T22:12:28,354 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/831adfb10d2e4296b01a22372f750435 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/831adfb10d2e4296b01a22372f750435 2024-11-15T22:12:28,356 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/cb98a1bcb69d4715a70081cd15be8dfc to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/cb98a1bcb69d4715a70081cd15be8dfc 2024-11-15T22:12:28,357 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/ccc00b1a79b345aa831ce0cbcaff21e0 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/ccc00b1a79b345aa831ce0cbcaff21e0 2024-11-15T22:12:28,358 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/f13eb1113eeb47b2a76f6a29ba305bbd to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/C/f13eb1113eeb47b2a76f6a29ba305bbd 2024-11-15T22:12:28,361 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/recovered.edits/487.seqid to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d/recovered.edits/487.seqid 2024-11-15T22:12:28,361 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,361 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T22:12:28,362 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T22:12:28,363 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T22:12:28,366 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111510b6e4ce0a2e4b55ad136383de49d746_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111510b6e4ce0a2e4b55ad136383de49d746_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,367 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115163e776ed7bc4c02ae8c83a883fa061d_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115163e776ed7bc4c02ae8c83a883fa061d_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,369 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111518d2c88fc0564569881b791d594a89e7_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111518d2c88fc0564569881b791d594a89e7_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,370 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152c978e7a159041eab7a195ca85fa50a5_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152c978e7a159041eab7a195ca85fa50a5_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,371 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115312ca372b928481a8835a38e05c685d1_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115312ca372b928481a8835a38e05c685d1_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,372 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115314a5ac9fb6945de8471f0ee2c24d6ff_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115314a5ac9fb6945de8471f0ee2c24d6ff_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,373 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115326bdd3e40974894a1e8464fac9fa1b7_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115326bdd3e40974894a1e8464fac9fa1b7_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,375 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153540accabec04f5883796d195d5767e8_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153540accabec04f5883796d195d5767e8_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,376 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153c8d0fc05b12409ba315b5b923299aba_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153c8d0fc05b12409ba315b5b923299aba_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,377 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111552d471c978f84380bd2a98d5d4f5d186_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111552d471c978f84380bd2a98d5d4f5d186_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,378 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111552f7877d5a664766a8adf050c4713a95_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111552f7877d5a664766a8adf050c4713a95_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,379 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111557e044d1456b487888120a412305810a_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111557e044d1456b487888120a412305810a_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,380 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115678f1303c39b472fa3a4a2a131a283ca_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115678f1303c39b472fa3a4a2a131a283ca_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,381 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156d6525df00ae43cf9fad748a0c4bc608_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156d6525df00ae43cf9fad748a0c4bc608_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,383 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157985b1f41f304df0a76390d3d3792eda_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157985b1f41f304df0a76390d3d3792eda_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,384 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115811bd788d82b47ab92e49456ade6b77b_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115811bd788d82b47ab92e49456ade6b77b_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,385 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115860542042b8b4edbb65f75b09c7062da_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115860542042b8b4edbb65f75b09c7062da_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,386 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159fa45affa68f4c849339e1a74e364d34_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159fa45affa68f4c849339e1a74e364d34_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,387 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a181402c43524c84b086fb87b87bcaaf_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a181402c43524c84b086fb87b87bcaaf_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,388 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a964a95b1d914ca78a1814a099fadb5f_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a964a95b1d914ca78a1814a099fadb5f_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,389 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115aa33a2da54364ab0850c3b6fd9e59863_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115aa33a2da54364ab0850c3b6fd9e59863_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,391 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b1409c8daafe4c2fb4d2fa5aad60b681_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b1409c8daafe4c2fb4d2fa5aad60b681_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,392 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115db8d182fde6e46f8ad680cf50bf551dd_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115db8d182fde6e46f8ad680cf50bf551dd_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,393 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e0d21cc6fb2347f28777b848649797ed_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e0d21cc6fb2347f28777b848649797ed_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,394 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f072b63155974ea099a6bc377c152192_2a24e00e990dba09c516fd169bc5958d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f072b63155974ea099a6bc377c152192_2a24e00e990dba09c516fd169bc5958d 2024-11-15T22:12:28,394 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T22:12:28,396 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:12:28,398 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T22:12:28,400 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T22:12:28,402 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:12:28,402 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T22:12:28,402 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731708748402"}]},"ts":"9223372036854775807"} 2024-11-15T22:12:28,404 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T22:12:28,404 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2a24e00e990dba09c516fd169bc5958d, NAME => 'TestAcidGuarantees,,1731708718204.2a24e00e990dba09c516fd169bc5958d.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T22:12:28,404 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T22:12:28,404 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731708748404"}]},"ts":"9223372036854775807"} 2024-11-15T22:12:28,405 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T22:12:28,406 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:12:28,407 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 69 msec 2024-11-15T22:12:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T22:12:28,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T22:12:28,446 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:28,457 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=241 (was 241), OpenFileDescriptor=461 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=699 (was 666) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2747 (was 2724) - AvailableMemoryMB LEAK? - 2024-11-15T22:12:28,466 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=241, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=699, ProcessCount=11, AvailableMemoryMB=2747 2024-11-15T22:12:28,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T22:12:28,468 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:12:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T22:12:28,470 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T22:12:28,470 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:28,470 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-15T22:12:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-15T22:12:28,471 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T22:12:28,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742056_1232 (size=959) 2024-11-15T22:12:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-15T22:12:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-15T22:12:28,879 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31 2024-11-15T22:12:28,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742057_1233 (size=53) 2024-11-15T22:12:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-15T22:12:29,292 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:12:29,292 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5d997b3c24773a2e95e850dd849536df, disabling compactions & flushes 2024-11-15T22:12:29,292 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:29,292 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:29,292 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. after waiting 0 ms 2024-11-15T22:12:29,292 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:29,292 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:29,292 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5d997b3c24773a2e95e850dd849536df: Waiting for close lock at 1731708749292Disabling compacts and flushes for region at 1731708749292Disabling writes for close at 1731708749292Writing region close event to WAL at 1731708749292Closed at 1731708749292 2024-11-15T22:12:29,293 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T22:12:29,294 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731708749294"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708749294"}]},"ts":"1731708749294"} 2024-11-15T22:12:29,296 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T22:12:29,300 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T22:12:29,300 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708749300"}]},"ts":"1731708749300"} 2024-11-15T22:12:29,302 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T22:12:29,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d997b3c24773a2e95e850dd849536df, ASSIGN}] 2024-11-15T22:12:29,304 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d997b3c24773a2e95e850dd849536df, ASSIGN 2024-11-15T22:12:29,305 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d997b3c24773a2e95e850dd849536df, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,37917,1731708686516; forceNewPlan=false, retain=false 2024-11-15T22:12:29,456 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=5d997b3c24773a2e95e850dd849536df, regionState=OPENING, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:29,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d997b3c24773a2e95e850dd849536df, ASSIGN because future has completed 2024-11-15T22:12:29,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516}] 2024-11-15T22:12:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-15T22:12:29,616 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:29,616 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:12:29,616 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:29,616 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:12:29,616 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:29,617 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:29,618 INFO [StoreOpener-5d997b3c24773a2e95e850dd849536df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:29,619 INFO [StoreOpener-5d997b3c24773a2e95e850dd849536df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d997b3c24773a2e95e850dd849536df columnFamilyName A 2024-11-15T22:12:29,619 DEBUG [StoreOpener-5d997b3c24773a2e95e850dd849536df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:29,620 INFO [StoreOpener-5d997b3c24773a2e95e850dd849536df-1 {}] regionserver.HStore(327): Store=5d997b3c24773a2e95e850dd849536df/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:12:29,620 INFO [StoreOpener-5d997b3c24773a2e95e850dd849536df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:29,621 INFO [StoreOpener-5d997b3c24773a2e95e850dd849536df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d997b3c24773a2e95e850dd849536df columnFamilyName B 2024-11-15T22:12:29,621 DEBUG [StoreOpener-5d997b3c24773a2e95e850dd849536df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:29,621 INFO [StoreOpener-5d997b3c24773a2e95e850dd849536df-1 {}] regionserver.HStore(327): Store=5d997b3c24773a2e95e850dd849536df/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:12:29,622 INFO [StoreOpener-5d997b3c24773a2e95e850dd849536df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:29,623 INFO [StoreOpener-5d997b3c24773a2e95e850dd849536df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d997b3c24773a2e95e850dd849536df columnFamilyName C 2024-11-15T22:12:29,623 DEBUG [StoreOpener-5d997b3c24773a2e95e850dd849536df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:12:29,623 INFO [StoreOpener-5d997b3c24773a2e95e850dd849536df-1 {}] regionserver.HStore(327): Store=5d997b3c24773a2e95e850dd849536df/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:12:29,623 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:29,624 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:29,624 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:29,625 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:29,625 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:29,625 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:12:29,626 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:29,628 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:12:29,628 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 5d997b3c24773a2e95e850dd849536df; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65745812, jitterRate=-0.020311057567596436}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:12:29,629 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:29,629 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 5d997b3c24773a2e95e850dd849536df: Running coprocessor pre-open hook at 1731708749617Writing region info on filesystem at 1731708749617Initializing all the Stores at 1731708749617Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708749617Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708749618 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708749618Cleaning up temporary data from old regions at 1731708749625 (+7 ms)Running coprocessor post-open hooks at 1731708749629 (+4 ms)Region opened successfully at 1731708749629 2024-11-15T22:12:29,630 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., pid=63, masterSystemTime=1731708749612 2024-11-15T22:12:29,632 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:29,632 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:29,632 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=5d997b3c24773a2e95e850dd849536df, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:29,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 because future has completed 2024-11-15T22:12:29,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-15T22:12:29,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 in 176 msec 2024-11-15T22:12:29,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-15T22:12:29,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d997b3c24773a2e95e850dd849536df, ASSIGN in 334 msec 2024-11-15T22:12:29,638 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T22:12:29,638 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708749638"}]},"ts":"1731708749638"} 2024-11-15T22:12:29,640 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T22:12:29,641 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T22:12:29,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1730 sec 2024-11-15T22:12:30,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-15T22:12:30,617 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:30,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13a2acf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,618 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:12:30,619 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:12:30,620 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:12:30,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:12:30,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:12:30,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@562bb7fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:12:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:12:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:30,622 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:12:30,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74281319, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,623 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:12:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:12:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:12:30,625 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:12:30,626 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T22:12:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is ae3311ea5ccb,37303,1731708685620 2024-11-15T22:12:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@539d75d1 2024-11-15T22:12:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T22:12:30,628 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T22:12:30,631 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@661204ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,631 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:12:30,631 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:12:30,631 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:12:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:12:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:12:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14c63abf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:12:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:12:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:30,633 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:12:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c67826, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40655b61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:12:30,635 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:12:30,635 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:12:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:12:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:12:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61fdcf59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:12:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:12:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:30,637 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:12:30,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a7874b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d355d6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,639 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:12:30,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:12:30,640 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:12:30,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:12:30,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:12:30,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75c955d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:12:30,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:12:30,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:30,641 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:12:30,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a79174b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf4401, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,643 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:12:30,643 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:12:30,644 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:12:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:12:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:12:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@181b4181, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:12:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:12:30,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:30,645 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:12:30,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cdcfffd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d01f46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:12:30,648 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:12:30,649 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:12:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:12:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:12:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f3612b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:12:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:12:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:30,650 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:12:30,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145eb33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a4b43fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,653 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:12:30,653 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:12:30,654 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:12:30,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:12:30,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:12:30,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67493b37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:12:30,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:12:30,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:30,656 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43718, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:12:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c698420, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aa78cca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,657 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:12:30,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:12:30,658 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:12:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:12:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:12:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c05137b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:12:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:12:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:30,659 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:12:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6966eefa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d06ee61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,661 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:12:30,661 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:12:30,662 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:12:30,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:12:30,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:12:30,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28eed94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:12:30,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:12:30,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:30,664 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:12:30,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cde10b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a31e52b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:12:30,665 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:12:30,666 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:12:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:12:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:12:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29ccdf55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:12:30,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:12:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:30,667 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:12:30,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a5e061b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ed5ef25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:12:30,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:12:30,670 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:12:30,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:12:30,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:12:30,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c734e65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:12:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:12:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:30,672 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:12:30,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2188e0df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:12:30,673 DEBUG [Thread-1093 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:12:30,674 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:12:30,674 DEBUG [Thread-1095 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:12:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:12:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:12:30,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:12:30,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:12:30,675 DEBUG [Thread-1104 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:12:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-15T22:12:30,676 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:12:30,676 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:12:30,676 DEBUG [Thread-1099 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:12:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T22:12:30,677 DEBUG [Thread-1106 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:12:30,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:12:30,677 DEBUG [Thread-1110 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:12:30,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:12:30,677 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:12:30,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:12:30,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:12:30,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:12:30,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:12:30,701 DEBUG [Thread-1112 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:12:30,702 DEBUG [Thread-1108 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:12:30,702 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:12:30,702 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:12:30,702 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:12:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:12:30,703 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:12:30,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:12:30,704 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:12:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:12:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:12:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:12:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:12:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:12:30,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:12:30,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:12:30,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:12:30,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:12:30,707 DEBUG [Thread-1097 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:12:30,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:12:30,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:12:30,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:12:30,712 DEBUG [Thread-1112 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:12:30,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:12:30,712 DEBUG [Thread-1112 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:12:30,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:12:30,713 DEBUG [Thread-1101 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:12:30,715 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:12:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:12:30,717 DEBUG [Thread-1101 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:12:30,717 DEBUG [Thread-1101 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:12:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:12:30,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:30,724 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:30,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T22:12:30,732 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:12:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:12:30,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/0e4d3f74131b44679ba1545f5e7e6bd2 is 50, key is test_row_0/A:col10/1731708750720/Put/seqid=0 2024-11-15T22:12:30,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T22:12:30,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708810779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:30,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:30,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:30,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:30,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:30,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708810779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:30,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:30,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708810782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:30,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:30,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708810787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:30,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:30,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:30,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:30,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708810789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:30,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:30,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:30,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:30,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742058_1234 (size=9657) 2024-11-15T22:12:30,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/0e4d3f74131b44679ba1545f5e7e6bd2 2024-11-15T22:12:30,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T22:12:30,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:30,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:30,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:30,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:30,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:30,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:30,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/515162c9010948bcac3123650d996bd2 is 50, key is test_row_0/B:col10/1731708750720/Put/seqid=0 2024-11-15T22:12:30,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:30,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708810898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708810898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:30,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:30,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708810898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708810898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:30,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708810899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:30,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742059_1235 (size=9657) 2024-11-15T22:12:30,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/515162c9010948bcac3123650d996bd2 2024-11-15T22:12:30,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/ca59894353b449e7bb0fca547cc2a034 is 50, key is test_row_0/C:col10/1731708750720/Put/seqid=0 2024-11-15T22:12:30,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742060_1236 (size=9657) 2024-11-15T22:12:30,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/ca59894353b449e7bb0fca547cc2a034 2024-11-15T22:12:30,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/0e4d3f74131b44679ba1545f5e7e6bd2 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/0e4d3f74131b44679ba1545f5e7e6bd2 2024-11-15T22:12:30,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/0e4d3f74131b44679ba1545f5e7e6bd2, entries=100, sequenceid=12, filesize=9.4 K 2024-11-15T22:12:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T22:12:31,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/515162c9010948bcac3123650d996bd2 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/515162c9010948bcac3123650d996bd2 2024-11-15T22:12:31,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/515162c9010948bcac3123650d996bd2, entries=100, sequenceid=12, filesize=9.4 K 2024-11-15T22:12:31,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/ca59894353b449e7bb0fca547cc2a034 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/ca59894353b449e7bb0fca547cc2a034 2024-11-15T22:12:31,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T22:12:31,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:31,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:31,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:31,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:31,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:31,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/ca59894353b449e7bb0fca547cc2a034, entries=100, sequenceid=12, filesize=9.4 K 2024-11-15T22:12:31,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5d997b3c24773a2e95e850dd849536df in 304ms, sequenceid=12, compaction requested=false 2024-11-15T22:12:31,028 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T22:12:31,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:31,118 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:31,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-15T22:12:31,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/49cbcb1b5d75415c86f1d373f22a6ef4 is 50, key is test_row_0/A:col10/1731708751107/Put/seqid=0 2024-11-15T22:12:31,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742061_1237 (size=12001) 2024-11-15T22:12:31,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/49cbcb1b5d75415c86f1d373f22a6ef4 2024-11-15T22:12:31,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T22:12:31,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:31,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:31,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:31,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:31,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:31,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708811173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708811179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708811179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708811182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708811186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/0a752d46426c49babcac33d8ece50ada is 50, key is test_row_0/B:col10/1731708751107/Put/seqid=0 2024-11-15T22:12:31,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742062_1238 (size=12001) 2024-11-15T22:12:31,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/0a752d46426c49babcac33d8ece50ada 2024-11-15T22:12:31,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/a58ad782868840fdb32939434273e3bb is 50, key is test_row_0/C:col10/1731708751107/Put/seqid=0 2024-11-15T22:12:31,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708811287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742063_1239 (size=12001) 2024-11-15T22:12:31,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/a58ad782868840fdb32939434273e3bb 2024-11-15T22:12:31,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708811296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708811296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708811296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708811297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/49cbcb1b5d75415c86f1d373f22a6ef4 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/49cbcb1b5d75415c86f1d373f22a6ef4 2024-11-15T22:12:31,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/49cbcb1b5d75415c86f1d373f22a6ef4, entries=150, sequenceid=41, filesize=11.7 K 2024-11-15T22:12:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T22:12:31,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/0a752d46426c49babcac33d8ece50ada as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/0a752d46426c49babcac33d8ece50ada 2024-11-15T22:12:31,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/0a752d46426c49babcac33d8ece50ada, entries=150, sequenceid=41, filesize=11.7 K 2024-11-15T22:12:31,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/a58ad782868840fdb32939434273e3bb as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/a58ad782868840fdb32939434273e3bb 2024-11-15T22:12:31,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/a58ad782868840fdb32939434273e3bb, entries=150, sequenceid=41, filesize=11.7 K 2024-11-15T22:12:31,324 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-15T22:12:31,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for 5d997b3c24773a2e95e850dd849536df in 207ms, sequenceid=41, compaction requested=false 2024-11-15T22:12:31,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:31,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T22:12:31,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:31,333 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T22:12:31,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/75932caeea48404088088c3f93189515 is 50, key is test_row_0/A:col10/1731708751175/Put/seqid=0 2024-11-15T22:12:31,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742064_1240 (size=12001) 2024-11-15T22:12:31,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:31,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708811542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708811543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708811545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708811545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708811547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708811657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708811657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708811657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708811657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708811657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,748 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/75932caeea48404088088c3f93189515 2024-11-15T22:12:31,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/6d0925f6ecd846a0ab897a92b0305298 is 50, key is test_row_0/B:col10/1731708751175/Put/seqid=0 2024-11-15T22:12:31,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742065_1241 (size=12001) 2024-11-15T22:12:31,784 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/6d0925f6ecd846a0ab897a92b0305298 2024-11-15T22:12:31,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/c2dd20c15c394b4aac6b5b3e5e792574 is 50, key is test_row_0/C:col10/1731708751175/Put/seqid=0 2024-11-15T22:12:31,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742066_1242 (size=12001) 2024-11-15T22:12:31,805 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/c2dd20c15c394b4aac6b5b3e5e792574 2024-11-15T22:12:31,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/75932caeea48404088088c3f93189515 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/75932caeea48404088088c3f93189515 2024-11-15T22:12:31,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T22:12:31,818 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/75932caeea48404088088c3f93189515, entries=150, sequenceid=49, filesize=11.7 K 2024-11-15T22:12:31,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/6d0925f6ecd846a0ab897a92b0305298 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/6d0925f6ecd846a0ab897a92b0305298 2024-11-15T22:12:31,827 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/6d0925f6ecd846a0ab897a92b0305298, entries=150, sequenceid=49, filesize=11.7 K 2024-11-15T22:12:31,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/c2dd20c15c394b4aac6b5b3e5e792574 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/c2dd20c15c394b4aac6b5b3e5e792574 2024-11-15T22:12:31,834 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/c2dd20c15c394b4aac6b5b3e5e792574, entries=150, sequenceid=49, filesize=11.7 K 2024-11-15T22:12:31,835 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 5d997b3c24773a2e95e850dd849536df in 502ms, sequenceid=49, compaction requested=true 2024-11-15T22:12:31,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:31,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:31,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-15T22:12:31,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-15T22:12:31,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-15T22:12:31,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1350 sec 2024-11-15T22:12:31,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.1680 sec 2024-11-15T22:12:31,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:31,869 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:31,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-15T22:12:31,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/0b64a02ef2164d05b9cd4a8877bdba95 is 50, key is test_row_0/A:col10/1731708751869/Put/seqid=0 2024-11-15T22:12:31,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742067_1243 (size=12001) 2024-11-15T22:12:31,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708811870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708811871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708811902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708811902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:31,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708811902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708812016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708812017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708812017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708812017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708812017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708812226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708812226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708812226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708812226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708812226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,243 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T22:12:32,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/0b64a02ef2164d05b9cd4a8877bdba95 2024-11-15T22:12:32,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/3d03efb84c604383a38547bbd9783c6e is 50, key is test_row_0/B:col10/1731708751869/Put/seqid=0 2024-11-15T22:12:32,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742068_1244 (size=12001) 2024-11-15T22:12:32,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/3d03efb84c604383a38547bbd9783c6e 2024-11-15T22:12:32,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/837b856eb1624800bd4c9b3d6d9bef99 is 50, key is test_row_0/C:col10/1731708751869/Put/seqid=0 2024-11-15T22:12:32,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742069_1245 (size=12001) 2024-11-15T22:12:32,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/837b856eb1624800bd4c9b3d6d9bef99 2024-11-15T22:12:32,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/0b64a02ef2164d05b9cd4a8877bdba95 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/0b64a02ef2164d05b9cd4a8877bdba95 2024-11-15T22:12:32,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/0b64a02ef2164d05b9cd4a8877bdba95, entries=150, sequenceid=79, filesize=11.7 K 2024-11-15T22:12:32,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/3d03efb84c604383a38547bbd9783c6e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/3d03efb84c604383a38547bbd9783c6e 2024-11-15T22:12:32,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/3d03efb84c604383a38547bbd9783c6e, entries=150, sequenceid=79, filesize=11.7 K 2024-11-15T22:12:32,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/837b856eb1624800bd4c9b3d6d9bef99 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/837b856eb1624800bd4c9b3d6d9bef99 2024-11-15T22:12:32,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/837b856eb1624800bd4c9b3d6d9bef99, entries=150, sequenceid=79, filesize=11.7 K 2024-11-15T22:12:32,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 5d997b3c24773a2e95e850dd849536df in 488ms, sequenceid=79, compaction requested=true 2024-11-15T22:12:32,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:32,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:12:32,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:32,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:12:32,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:12:32,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:12:32,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T22:12:32,358 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:12:32,358 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:12:32,361 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:12:32,361 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/A is initiating minor compaction (all files) 2024-11-15T22:12:32,361 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:12:32,361 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/A in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:32,361 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/C is initiating minor compaction (all files) 2024-11-15T22:12:32,361 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/C in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:32,361 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/0e4d3f74131b44679ba1545f5e7e6bd2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/49cbcb1b5d75415c86f1d373f22a6ef4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/75932caeea48404088088c3f93189515, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/0b64a02ef2164d05b9cd4a8877bdba95] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=44.6 K 2024-11-15T22:12:32,361 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/ca59894353b449e7bb0fca547cc2a034, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/a58ad782868840fdb32939434273e3bb, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/c2dd20c15c394b4aac6b5b3e5e792574, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/837b856eb1624800bd4c9b3d6d9bef99] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=44.6 K 2024-11-15T22:12:32,362 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e4d3f74131b44679ba1545f5e7e6bd2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731708750720 2024-11-15T22:12:32,362 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting ca59894353b449e7bb0fca547cc2a034, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731708750720 2024-11-15T22:12:32,362 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting a58ad782868840fdb32939434273e3bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731708751107 2024-11-15T22:12:32,362 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49cbcb1b5d75415c86f1d373f22a6ef4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731708751107 2024-11-15T22:12:32,363 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75932caeea48404088088c3f93189515, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731708751125 2024-11-15T22:12:32,363 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting c2dd20c15c394b4aac6b5b3e5e792574, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731708751125 2024-11-15T22:12:32,364 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 837b856eb1624800bd4c9b3d6d9bef99, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731708751543 2024-11-15T22:12:32,364 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b64a02ef2164d05b9cd4a8877bdba95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731708751543 2024-11-15T22:12:32,382 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#C#compaction#194 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:32,384 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/0b03fe7d1c7a4941b914f81b49e82518 is 50, key is test_row_0/C:col10/1731708751869/Put/seqid=0 2024-11-15T22:12:32,392 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#A#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:32,392 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/723b7bcb1e294788a71116aa7093a82d is 50, key is test_row_0/A:col10/1731708751869/Put/seqid=0 2024-11-15T22:12:32,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742070_1246 (size=12139) 2024-11-15T22:12:32,411 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/0b03fe7d1c7a4941b914f81b49e82518 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/0b03fe7d1c7a4941b914f81b49e82518 2024-11-15T22:12:32,417 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/C of 5d997b3c24773a2e95e850dd849536df into 0b03fe7d1c7a4941b914f81b49e82518(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:32,417 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:32,417 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/C, priority=12, startTime=1731708752358; duration=0sec 2024-11-15T22:12:32,417 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:32,417 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:C 2024-11-15T22:12:32,418 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:12:32,419 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:12:32,420 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/B is initiating minor compaction (all files) 2024-11-15T22:12:32,420 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/B in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:32,420 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/515162c9010948bcac3123650d996bd2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/0a752d46426c49babcac33d8ece50ada, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/6d0925f6ecd846a0ab897a92b0305298, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/3d03efb84c604383a38547bbd9783c6e] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=44.6 K 2024-11-15T22:12:32,420 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 515162c9010948bcac3123650d996bd2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731708750720 2024-11-15T22:12:32,421 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a752d46426c49babcac33d8ece50ada, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731708751107 2024-11-15T22:12:32,421 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d0925f6ecd846a0ab897a92b0305298, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731708751125 2024-11-15T22:12:32,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742071_1247 (size=12139) 2024-11-15T22:12:32,422 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d03efb84c604383a38547bbd9783c6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731708751543 2024-11-15T22:12:32,433 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/723b7bcb1e294788a71116aa7093a82d as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/723b7bcb1e294788a71116aa7093a82d 2024-11-15T22:12:32,444 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/A of 5d997b3c24773a2e95e850dd849536df into 723b7bcb1e294788a71116aa7093a82d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:32,444 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:32,444 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/A, priority=12, startTime=1731708752357; duration=0sec 2024-11-15T22:12:32,444 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:32,444 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:A 2024-11-15T22:12:32,446 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#B#compaction#196 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:32,446 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/dac37fe4251e47e7a75ad336f116c2ef is 50, key is test_row_0/B:col10/1731708751869/Put/seqid=0 2024-11-15T22:12:32,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742072_1248 (size=12139) 2024-11-15T22:12:32,472 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/dac37fe4251e47e7a75ad336f116c2ef as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/dac37fe4251e47e7a75ad336f116c2ef 2024-11-15T22:12:32,479 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/B of 5d997b3c24773a2e95e850dd849536df into dac37fe4251e47e7a75ad336f116c2ef(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:32,480 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:32,480 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/B, priority=12, startTime=1731708752357; duration=0sec 2024-11-15T22:12:32,480 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:32,480 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:B 2024-11-15T22:12:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:32,542 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:32,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T22:12:32,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/14208973c8e54e39a53309f612071651 is 50, key is test_row_0/A:col10/1731708752537/Put/seqid=0 2024-11-15T22:12:32,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742073_1249 (size=12001) 2024-11-15T22:12:32,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708812571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708812571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708812571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708812577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708812578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708812687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708812687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708812687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708812687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708812687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T22:12:32,837 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:32,837 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:12:32,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-15T22:12:32,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T22:12:32,845 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:12:32,846 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:12:32,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:12:32,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708812897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708812897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708812897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708812897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708812897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T22:12:32,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/14208973c8e54e39a53309f612071651 2024-11-15T22:12:32,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/1b16e3b3f6264e4bb62b3232f242ee61 is 50, key is test_row_0/B:col10/1731708752537/Put/seqid=0 2024-11-15T22:12:32,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742074_1250 (size=12001) 2024-11-15T22:12:32,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/1b16e3b3f6264e4bb62b3232f242ee61 2024-11-15T22:12:32,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T22:12:32,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:32,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:32,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:32,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:33,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:33,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/9fd49f1133214459817687985eebd900 is 50, key is test_row_0/C:col10/1731708752537/Put/seqid=0 2024-11-15T22:12:33,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742075_1251 (size=12001) 2024-11-15T22:12:33,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/9fd49f1133214459817687985eebd900 2024-11-15T22:12:33,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/14208973c8e54e39a53309f612071651 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/14208973c8e54e39a53309f612071651 2024-11-15T22:12:33,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/14208973c8e54e39a53309f612071651, entries=150, sequenceid=93, filesize=11.7 K 2024-11-15T22:12:33,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/1b16e3b3f6264e4bb62b3232f242ee61 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/1b16e3b3f6264e4bb62b3232f242ee61 2024-11-15T22:12:33,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/1b16e3b3f6264e4bb62b3232f242ee61, entries=150, sequenceid=93, filesize=11.7 K 2024-11-15T22:12:33,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/9fd49f1133214459817687985eebd900 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/9fd49f1133214459817687985eebd900 2024-11-15T22:12:33,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/9fd49f1133214459817687985eebd900, entries=150, sequenceid=93, filesize=11.7 K 2024-11-15T22:12:33,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5d997b3c24773a2e95e850dd849536df in 498ms, sequenceid=93, compaction requested=false 2024-11-15T22:12:33,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:33,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T22:12:33,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:33,153 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T22:12:33,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T22:12:33,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/a468bf814c5946c890fc01e546bf3e79 is 50, key is test_row_0/A:col10/1731708752562/Put/seqid=0 2024-11-15T22:12:33,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742076_1252 (size=12001) 2024-11-15T22:12:33,178 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/a468bf814c5946c890fc01e546bf3e79 2024-11-15T22:12:33,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/92ffb9df02ea4ac7b17ec1cdfe4892f2 is 50, key is test_row_0/B:col10/1731708752562/Put/seqid=0 2024-11-15T22:12:33,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:33,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:33,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742077_1253 (size=12001) 2024-11-15T22:12:33,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:33,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708813224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:33,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708813224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:33,233 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/92ffb9df02ea4ac7b17ec1cdfe4892f2 2024-11-15T22:12:33,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:33,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708813228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:33,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:33,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:33,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708813230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:33,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:33,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:33,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708813235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:33,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:33,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/c03829331ee94090a10c030d02275672 is 50, key is test_row_0/C:col10/1731708752562/Put/seqid=0 2024-11-15T22:12:33,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742078_1254 (size=12001) 2024-11-15T22:12:33,256 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/c03829331ee94090a10c030d02275672 2024-11-15T22:12:33,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/a468bf814c5946c890fc01e546bf3e79 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/a468bf814c5946c890fc01e546bf3e79 2024-11-15T22:12:33,273 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/a468bf814c5946c890fc01e546bf3e79, entries=150, sequenceid=118, filesize=11.7 K 2024-11-15T22:12:33,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/92ffb9df02ea4ac7b17ec1cdfe4892f2 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/92ffb9df02ea4ac7b17ec1cdfe4892f2 2024-11-15T22:12:33,283 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/92ffb9df02ea4ac7b17ec1cdfe4892f2, entries=150, sequenceid=118, filesize=11.7 K 2024-11-15T22:12:33,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/c03829331ee94090a10c030d02275672 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/c03829331ee94090a10c030d02275672 2024-11-15T22:12:33,290 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/c03829331ee94090a10c030d02275672, entries=150, sequenceid=118, filesize=11.7 K 2024-11-15T22:12:33,292 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5d997b3c24773a2e95e850dd849536df in 138ms, sequenceid=118, compaction requested=true 2024-11-15T22:12:33,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:33,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:33,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-15T22:12:33,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-15T22:12:33,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-15T22:12:33,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 447 msec 2024-11-15T22:12:33,298 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 460 msec 2024-11-15T22:12:33,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:33,339 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:33,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T22:12:33,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/c60c5b1a253047e28cbe6608f34e36a2 is 50, key is test_row_0/A:col10/1731708753339/Put/seqid=0 2024-11-15T22:12:33,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742079_1255 (size=16831) 2024-11-15T22:12:33,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/c60c5b1a253047e28cbe6608f34e36a2 2024-11-15T22:12:33,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:33,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:33,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708813375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:33,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708813374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:33,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:33,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708813377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:33,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:33,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708813379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:33,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:33,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708813384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:33,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/d1b6a592bc074b01a28c817bf6e734da is 50, key is test_row_0/B:col10/1731708753339/Put/seqid=0 2024-11-15T22:12:33,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742080_1256 (size=12101) 2024-11-15T22:12:33,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T22:12:33,468 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:33,469 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:12:33,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-15T22:12:33,471 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:12:33,472 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:12:33,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:12:33,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T22:12:33,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:33,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708813487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:33,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:33,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:33,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708813487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:33,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:33,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:33,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708813487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:33,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:33,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708813497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:33,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708813501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:33,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T22:12:33,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T22:12:33,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:33,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:33,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:33,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:33,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:33,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:33,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708813697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:33,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:33,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708813697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:33,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:33,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:33,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708813697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:33,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:33,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:33,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708813706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:33,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:33,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708813716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:33,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T22:12:33,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:33,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:33,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:33,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:33,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:33,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:33,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T22:12:33,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/d1b6a592bc074b01a28c817bf6e734da 2024-11-15T22:12:33,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/5de3b874e7f6413f833390fc0232a724 is 50, key is test_row_0/C:col10/1731708753339/Put/seqid=0 2024-11-15T22:12:33,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742081_1257 (size=12101) 2024-11-15T22:12:33,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T22:12:33,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:33,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:33,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:33,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:33,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:34,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708814007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:34,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:34,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708814007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:34,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708814007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:34,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:34,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:34,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:34,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708814016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:34,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:34,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708814026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:34,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:34,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T22:12:34,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:34,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:34,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:34,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:34,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:34,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:34,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T22:12:34,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T22:12:34,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:34,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:34,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:34,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:34,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:34,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:34,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/5de3b874e7f6413f833390fc0232a724 2024-11-15T22:12:34,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/c60c5b1a253047e28cbe6608f34e36a2 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/c60c5b1a253047e28cbe6608f34e36a2 2024-11-15T22:12:34,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/c60c5b1a253047e28cbe6608f34e36a2, entries=250, sequenceid=132, filesize=16.4 K 2024-11-15T22:12:34,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/d1b6a592bc074b01a28c817bf6e734da as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/d1b6a592bc074b01a28c817bf6e734da 2024-11-15T22:12:34,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/d1b6a592bc074b01a28c817bf6e734da, entries=150, sequenceid=132, filesize=11.8 K 2024-11-15T22:12:34,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/5de3b874e7f6413f833390fc0232a724 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/5de3b874e7f6413f833390fc0232a724 2024-11-15T22:12:34,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/5de3b874e7f6413f833390fc0232a724, entries=150, sequenceid=132, filesize=11.8 K 2024-11-15T22:12:34,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 5d997b3c24773a2e95e850dd849536df in 979ms, sequenceid=132, compaction requested=true 2024-11-15T22:12:34,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:34,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:12:34,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:34,319 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:12:34,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:12:34,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:34,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:12:34,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:34,319 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:12:34,320 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:12:34,320 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52972 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:12:34,320 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/B is initiating minor compaction (all files) 2024-11-15T22:12:34,320 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/A is initiating minor compaction (all files) 2024-11-15T22:12:34,320 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/B in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:34,320 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/A in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:34,321 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/dac37fe4251e47e7a75ad336f116c2ef, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/1b16e3b3f6264e4bb62b3232f242ee61, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/92ffb9df02ea4ac7b17ec1cdfe4892f2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/d1b6a592bc074b01a28c817bf6e734da] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=47.1 K 2024-11-15T22:12:34,321 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/723b7bcb1e294788a71116aa7093a82d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/14208973c8e54e39a53309f612071651, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/a468bf814c5946c890fc01e546bf3e79, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/c60c5b1a253047e28cbe6608f34e36a2] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=51.7 K 2024-11-15T22:12:34,321 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting dac37fe4251e47e7a75ad336f116c2ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731708751543 2024-11-15T22:12:34,321 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 723b7bcb1e294788a71116aa7093a82d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731708751543 2024-11-15T22:12:34,321 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14208973c8e54e39a53309f612071651, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731708752537 2024-11-15T22:12:34,321 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b16e3b3f6264e4bb62b3232f242ee61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731708752537 2024-11-15T22:12:34,322 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting a468bf814c5946c890fc01e546bf3e79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731708752562 2024-11-15T22:12:34,322 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 92ffb9df02ea4ac7b17ec1cdfe4892f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731708752562 2024-11-15T22:12:34,322 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting d1b6a592bc074b01a28c817bf6e734da, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731708753228 2024-11-15T22:12:34,322 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting c60c5b1a253047e28cbe6608f34e36a2, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731708753217 2024-11-15T22:12:34,336 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#B#compaction#206 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:34,336 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/40836c91f99d465ab1edc2a987974ef3 is 50, key is test_row_0/B:col10/1731708753339/Put/seqid=0 2024-11-15T22:12:34,336 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#A#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:34,337 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/a352c4a0854b43a79f61324a7054f0e8 is 50, key is test_row_0/A:col10/1731708753339/Put/seqid=0 2024-11-15T22:12:34,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742082_1258 (size=12375) 2024-11-15T22:12:34,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742083_1259 (size=12375) 2024-11-15T22:12:34,401 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/a352c4a0854b43a79f61324a7054f0e8 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/a352c4a0854b43a79f61324a7054f0e8 2024-11-15T22:12:34,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T22:12:34,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:34,404 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T22:12:34,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/bde33d909ea049c4b55b8a608db8f2ed is 50, key is test_row_0/A:col10/1731708753375/Put/seqid=0 2024-11-15T22:12:34,416 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/A of 5d997b3c24773a2e95e850dd849536df into a352c4a0854b43a79f61324a7054f0e8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:34,416 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:34,416 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/A, priority=12, startTime=1731708754319; duration=0sec 2024-11-15T22:12:34,417 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:34,417 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:A 2024-11-15T22:12:34,417 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:12:34,419 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:12:34,419 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/C is initiating minor compaction (all files) 2024-11-15T22:12:34,420 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/C in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:34,420 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/0b03fe7d1c7a4941b914f81b49e82518, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/9fd49f1133214459817687985eebd900, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/c03829331ee94090a10c030d02275672, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/5de3b874e7f6413f833390fc0232a724] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=47.1 K 2024-11-15T22:12:34,420 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b03fe7d1c7a4941b914f81b49e82518, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731708751543 2024-11-15T22:12:34,421 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fd49f1133214459817687985eebd900, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731708752537 2024-11-15T22:12:34,421 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting c03829331ee94090a10c030d02275672, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731708752562 2024-11-15T22:12:34,422 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5de3b874e7f6413f833390fc0232a724, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731708753228 2024-11-15T22:12:34,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742084_1260 (size=12151) 2024-11-15T22:12:34,424 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/bde33d909ea049c4b55b8a608db8f2ed 2024-11-15T22:12:34,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/b265d3fbb04341c2aa56668f11d0437a is 50, key is test_row_0/B:col10/1731708753375/Put/seqid=0 2024-11-15T22:12:34,471 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#C#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:34,472 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/7ec9644dab454995b88dd77b409649ab is 50, key is test_row_0/C:col10/1731708753339/Put/seqid=0 2024-11-15T22:12:34,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742085_1261 (size=12151) 2024-11-15T22:12:34,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742086_1262 (size=12375) 2024-11-15T22:12:34,520 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/7ec9644dab454995b88dd77b409649ab as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/7ec9644dab454995b88dd77b409649ab 2024-11-15T22:12:34,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:34,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:34,526 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/C of 5d997b3c24773a2e95e850dd849536df into 7ec9644dab454995b88dd77b409649ab(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:34,526 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:34,526 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/C, priority=12, startTime=1731708754319; duration=0sec 2024-11-15T22:12:34,527 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:34,527 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:C 2024-11-15T22:12:34,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708814536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708814536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:34,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708814537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:34,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708814538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:34,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708814540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:34,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:34,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T22:12:34,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708814646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:34,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708814646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:34,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708814647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:34,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708814657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:34,770 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/40836c91f99d465ab1edc2a987974ef3 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/40836c91f99d465ab1edc2a987974ef3 2024-11-15T22:12:34,775 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/B of 5d997b3c24773a2e95e850dd849536df into 40836c91f99d465ab1edc2a987974ef3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:34,775 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:34,775 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/B, priority=12, startTime=1731708754319; duration=0sec 2024-11-15T22:12:34,775 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:34,775 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:B 2024-11-15T22:12:34,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708814856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:34,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708814857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:34,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708814857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:34,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708814866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:34,887 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/b265d3fbb04341c2aa56668f11d0437a 2024-11-15T22:12:34,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/81b500ab8a3243d3b1b6877d7c627f9e is 50, key is test_row_0/C:col10/1731708753375/Put/seqid=0 2024-11-15T22:12:34,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742087_1263 (size=12151) 2024-11-15T22:12:35,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708815166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:35,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708815166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:35,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708815166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:35,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:35,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:35,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708815177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:35,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:35,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:35,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:35,304 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/81b500ab8a3243d3b1b6877d7c627f9e 2024-11-15T22:12:35,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/bde33d909ea049c4b55b8a608db8f2ed as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/bde33d909ea049c4b55b8a608db8f2ed 2024-11-15T22:12:35,314 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/bde33d909ea049c4b55b8a608db8f2ed, entries=150, sequenceid=154, filesize=11.9 K 2024-11-15T22:12:35,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/b265d3fbb04341c2aa56668f11d0437a as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/b265d3fbb04341c2aa56668f11d0437a 2024-11-15T22:12:35,319 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/b265d3fbb04341c2aa56668f11d0437a, entries=150, sequenceid=154, filesize=11.9 K 2024-11-15T22:12:35,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/81b500ab8a3243d3b1b6877d7c627f9e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/81b500ab8a3243d3b1b6877d7c627f9e 2024-11-15T22:12:35,325 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/81b500ab8a3243d3b1b6877d7c627f9e, entries=150, sequenceid=154, filesize=11.9 K 2024-11-15T22:12:35,325 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 5d997b3c24773a2e95e850dd849536df in 921ms, sequenceid=154, compaction requested=false 2024-11-15T22:12:35,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:35,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:35,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-15T22:12:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-15T22:12:35,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-15T22:12:35,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8550 sec 2024-11-15T22:12:35,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.8610 sec 2024-11-15T22:12:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:35,560 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:35,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T22:12:35,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/dcc963b6d386421e833002af4a7c68e1 is 50, key is test_row_0/A:col10/1731708754531/Put/seqid=0 2024-11-15T22:12:35,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742088_1264 (size=12151) 2024-11-15T22:12:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T22:12:35,617 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:35,617 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:12:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-15T22:12:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T22:12:35,624 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:12:35,625 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:12:35,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:12:35,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708815670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:35,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708815676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:35,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:35,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:35,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:35,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708815687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:35,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:35,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:35,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:35,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708815687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:35,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:35,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:35,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:35,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708815687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:35,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:35,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:35,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T22:12:35,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T22:12:35,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:35,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:35,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:35,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:35,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:35,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708815786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:35,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:35,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:35,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:35,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T22:12:35,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:35,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:35,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:35,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:35,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T22:12:36,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708815997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:36,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:36,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:36,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:36,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/dcc963b6d386421e833002af4a7c68e1 2024-11-15T22:12:36,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/4e8e8fc6250649c6ad0dce3827faf4a0 is 50, key is test_row_0/B:col10/1731708754531/Put/seqid=0 2024-11-15T22:12:36,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742089_1265 (size=12151) 2024-11-15T22:12:36,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T22:12:36,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:36,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:36,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:36,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:36,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:36,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T22:12:36,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:36,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:36,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:36,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:36,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:36,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T22:12:36,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708816307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:36,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:36,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:36,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:36,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T22:12:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:36,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:36,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:36,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/4e8e8fc6250649c6ad0dce3827faf4a0 2024-11-15T22:12:36,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/9dfcc9d4e0ee47c3be00d58f59bfaa3a is 50, key is test_row_0/C:col10/1731708754531/Put/seqid=0 2024-11-15T22:12:36,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742090_1266 (size=12151) 2024-11-15T22:12:36,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T22:12:36,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:36,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:36,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:36,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:36,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:36,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708816697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:36,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708816697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:36,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708816698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:36,701 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_1' in 5d997b3c24773a2e95e850dd849536df of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:36,701 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_2' in 5d997b3c24773a2e95e850dd849536df of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:36,702 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_1' in 5d997b3c24773a2e95e850dd849536df of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:36,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708816709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:36,715 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_0' in 5d997b3c24773a2e95e850dd849536df of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:36,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T22:12:36,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:36,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:36,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:36,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:36,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T22:12:36,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708816817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:36,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T22:12:36,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:36,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:36,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:36,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:36,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:36,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:36,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/9dfcc9d4e0ee47c3be00d58f59bfaa3a 2024-11-15T22:12:36,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/dcc963b6d386421e833002af4a7c68e1 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/dcc963b6d386421e833002af4a7c68e1 2024-11-15T22:12:36,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/dcc963b6d386421e833002af4a7c68e1, entries=150, sequenceid=172, filesize=11.9 K 2024-11-15T22:12:36,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/4e8e8fc6250649c6ad0dce3827faf4a0 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4e8e8fc6250649c6ad0dce3827faf4a0 2024-11-15T22:12:36,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4e8e8fc6250649c6ad0dce3827faf4a0, entries=150, sequenceid=172, filesize=11.9 K 2024-11-15T22:12:36,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/9dfcc9d4e0ee47c3be00d58f59bfaa3a as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/9dfcc9d4e0ee47c3be00d58f59bfaa3a 2024-11-15T22:12:36,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/9dfcc9d4e0ee47c3be00d58f59bfaa3a, entries=150, sequenceid=172, filesize=11.9 K 2024-11-15T22:12:36,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 5d997b3c24773a2e95e850dd849536df in 1394ms, sequenceid=172, compaction requested=true 2024-11-15T22:12:36,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:36,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:12:36,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:36,955 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:36,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:12:36,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:36,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:12:36,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:36,955 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:36,959 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:36,959 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:36,959 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/A is initiating minor compaction (all files) 2024-11-15T22:12:36,959 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/B is initiating minor compaction (all files) 2024-11-15T22:12:36,959 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/A in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:36,959 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/B in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:36,959 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/40836c91f99d465ab1edc2a987974ef3, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/b265d3fbb04341c2aa56668f11d0437a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4e8e8fc6250649c6ad0dce3827faf4a0] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=35.8 K 2024-11-15T22:12:36,959 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/a352c4a0854b43a79f61324a7054f0e8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/bde33d909ea049c4b55b8a608db8f2ed, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/dcc963b6d386421e833002af4a7c68e1] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=35.8 K 2024-11-15T22:12:36,960 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting a352c4a0854b43a79f61324a7054f0e8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731708753228 2024-11-15T22:12:36,960 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 40836c91f99d465ab1edc2a987974ef3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731708753228 2024-11-15T22:12:36,961 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting b265d3fbb04341c2aa56668f11d0437a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731708753364 2024-11-15T22:12:36,961 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting bde33d909ea049c4b55b8a608db8f2ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731708753364 2024-11-15T22:12:36,961 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e8e8fc6250649c6ad0dce3827faf4a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731708754531 2024-11-15T22:12:36,961 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcc963b6d386421e833002af4a7c68e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731708754531 2024-11-15T22:12:36,984 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#B#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:36,985 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/752b6c46d0194e508044122687e4c56f is 50, key is test_row_0/B:col10/1731708754531/Put/seqid=0 2024-11-15T22:12:36,987 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#A#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:36,988 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/7673ee6f673d45b288eb81f73296d3f6 is 50, key is test_row_0/A:col10/1731708754531/Put/seqid=0 2024-11-15T22:12:37,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742091_1267 (size=12527) 2024-11-15T22:12:37,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742092_1268 (size=12527) 2024-11-15T22:12:37,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T22:12:37,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:37,042 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T22:12:37,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/963e7989ff5e4fdf94c306eed846ff04 is 50, key is test_row_0/A:col10/1731708755650/Put/seqid=0 2024-11-15T22:12:37,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742093_1269 (size=12151) 2024-11-15T22:12:37,427 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/752b6c46d0194e508044122687e4c56f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/752b6c46d0194e508044122687e4c56f 2024-11-15T22:12:37,431 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/7673ee6f673d45b288eb81f73296d3f6 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/7673ee6f673d45b288eb81f73296d3f6 2024-11-15T22:12:37,434 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/B of 5d997b3c24773a2e95e850dd849536df into 752b6c46d0194e508044122687e4c56f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:37,434 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:37,434 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/B, priority=13, startTime=1731708756955; duration=0sec 2024-11-15T22:12:37,434 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:37,434 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:B 2024-11-15T22:12:37,434 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:37,437 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:37,438 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/C is initiating minor compaction (all files) 2024-11-15T22:12:37,438 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/C in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:37,438 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/7ec9644dab454995b88dd77b409649ab, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/81b500ab8a3243d3b1b6877d7c627f9e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/9dfcc9d4e0ee47c3be00d58f59bfaa3a] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=35.8 K 2024-11-15T22:12:37,439 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ec9644dab454995b88dd77b409649ab, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731708753228 2024-11-15T22:12:37,439 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/A of 5d997b3c24773a2e95e850dd849536df into 7673ee6f673d45b288eb81f73296d3f6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:37,439 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:37,439 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/A, priority=13, startTime=1731708756955; duration=0sec 2024-11-15T22:12:37,439 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 81b500ab8a3243d3b1b6877d7c627f9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731708753364 2024-11-15T22:12:37,439 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:37,439 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:A 2024-11-15T22:12:37,440 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dfcc9d4e0ee47c3be00d58f59bfaa3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731708754531 2024-11-15T22:12:37,447 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#C#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:37,447 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/6c9acbbce1094ff0b44fef572f44d04f is 50, key is test_row_0/C:col10/1731708754531/Put/seqid=0 2024-11-15T22:12:37,455 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/963e7989ff5e4fdf94c306eed846ff04 2024-11-15T22:12:37,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742094_1270 (size=12527) 2024-11-15T22:12:37,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/7c6fae07d4074fc7a88649f26a0c3d46 is 50, key is test_row_0/B:col10/1731708755650/Put/seqid=0 2024-11-15T22:12:37,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742095_1271 (size=12151) 2024-11-15T22:12:37,476 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/7c6fae07d4074fc7a88649f26a0c3d46 2024-11-15T22:12:37,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/68840900fe6d4a24a6e7447d63951aee is 50, key is test_row_0/C:col10/1731708755650/Put/seqid=0 2024-11-15T22:12:37,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742096_1272 (size=12151) 2024-11-15T22:12:37,491 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/68840900fe6d4a24a6e7447d63951aee 2024-11-15T22:12:37,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/963e7989ff5e4fdf94c306eed846ff04 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/963e7989ff5e4fdf94c306eed846ff04 2024-11-15T22:12:37,505 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/963e7989ff5e4fdf94c306eed846ff04, entries=150, sequenceid=193, filesize=11.9 K 2024-11-15T22:12:37,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/7c6fae07d4074fc7a88649f26a0c3d46 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/7c6fae07d4074fc7a88649f26a0c3d46 2024-11-15T22:12:37,513 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/7c6fae07d4074fc7a88649f26a0c3d46, entries=150, sequenceid=193, filesize=11.9 K 2024-11-15T22:12:37,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/68840900fe6d4a24a6e7447d63951aee as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/68840900fe6d4a24a6e7447d63951aee 2024-11-15T22:12:37,530 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/68840900fe6d4a24a6e7447d63951aee, entries=150, sequenceid=193, filesize=11.9 K 2024-11-15T22:12:37,531 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for 5d997b3c24773a2e95e850dd849536df in 489ms, sequenceid=193, compaction requested=false 2024-11-15T22:12:37,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:37,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:37,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-15T22:12:37,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-15T22:12:37,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-15T22:12:37,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9090 sec 2024-11-15T22:12:37,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.9220 sec 2024-11-15T22:12:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T22:12:37,777 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:37,778 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:12:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-15T22:12:37,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T22:12:37,780 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:12:37,781 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:12:37,781 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:12:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:37,860 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:37,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T22:12:37,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/06578bf97e2f4726813df336bd5bfa6c is 50, key is test_row_0/A:col10/1731708757849/Put/seqid=0 2024-11-15T22:12:37,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742097_1273 (size=14541) 2024-11-15T22:12:37,871 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/6c9acbbce1094ff0b44fef572f44d04f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/6c9acbbce1094ff0b44fef572f44d04f 2024-11-15T22:12:37,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/06578bf97e2f4726813df336bd5bfa6c 2024-11-15T22:12:37,877 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/C of 5d997b3c24773a2e95e850dd849536df into 6c9acbbce1094ff0b44fef572f44d04f(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:37,877 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:37,877 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/C, priority=13, startTime=1731708756955; duration=0sec 2024-11-15T22:12:37,877 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:37,877 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:C 2024-11-15T22:12:37,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/177b842d6ab94367af98a89b7debb4a7 is 50, key is test_row_0/B:col10/1731708757849/Put/seqid=0 2024-11-15T22:12:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T22:12:37,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742098_1274 (size=12151) 2024-11-15T22:12:37,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/177b842d6ab94367af98a89b7debb4a7 2024-11-15T22:12:37,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/12a97b8cf6514472818a2c0cc718aec6 is 50, key is test_row_0/C:col10/1731708757849/Put/seqid=0 2024-11-15T22:12:37,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742099_1275 (size=12151) 2024-11-15T22:12:37,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/12a97b8cf6514472818a2c0cc718aec6 2024-11-15T22:12:37,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/06578bf97e2f4726813df336bd5bfa6c as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/06578bf97e2f4726813df336bd5bfa6c 2024-11-15T22:12:37,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/06578bf97e2f4726813df336bd5bfa6c, entries=200, sequenceid=206, filesize=14.2 K 2024-11-15T22:12:37,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/177b842d6ab94367af98a89b7debb4a7 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/177b842d6ab94367af98a89b7debb4a7 2024-11-15T22:12:37,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/177b842d6ab94367af98a89b7debb4a7, entries=150, sequenceid=206, filesize=11.9 K 2024-11-15T22:12:37,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/12a97b8cf6514472818a2c0cc718aec6 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/12a97b8cf6514472818a2c0cc718aec6 2024-11-15T22:12:37,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/12a97b8cf6514472818a2c0cc718aec6, entries=150, sequenceid=206, filesize=11.9 K 2024-11-15T22:12:37,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=80.51 KB/82440 for 5d997b3c24773a2e95e850dd849536df in 72ms, sequenceid=206, compaction requested=true 2024-11-15T22:12:37,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:37,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:12:37,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:37,932 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:37,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:12:37,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:37,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:12:37,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:37,933 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:37,934 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:37,934 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/A is initiating minor compaction (all files) 2024-11-15T22:12:37,934 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/A in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:37,934 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/7673ee6f673d45b288eb81f73296d3f6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/963e7989ff5e4fdf94c306eed846ff04, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/06578bf97e2f4726813df336bd5bfa6c] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=38.3 K 2024-11-15T22:12:37,934 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:37,934 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7673ee6f673d45b288eb81f73296d3f6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731708754531 2024-11-15T22:12:37,934 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/B is initiating minor compaction (all files) 2024-11-15T22:12:37,934 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/B in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:37,935 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/752b6c46d0194e508044122687e4c56f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/7c6fae07d4074fc7a88649f26a0c3d46, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/177b842d6ab94367af98a89b7debb4a7] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=36.0 K 2024-11-15T22:12:37,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T22:12:37,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:37,936 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T22:12:37,936 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 752b6c46d0194e508044122687e4c56f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731708754531 2024-11-15T22:12:37,936 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 963e7989ff5e4fdf94c306eed846ff04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731708755650 2024-11-15T22:12:37,936 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c6fae07d4074fc7a88649f26a0c3d46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731708755650 2024-11-15T22:12:37,937 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06578bf97e2f4726813df336bd5bfa6c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731708757844 2024-11-15T22:12:37,937 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 177b842d6ab94367af98a89b7debb4a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731708757849 2024-11-15T22:12:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:37,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:37,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/c45a2916b4144987a622a479db3c5451 is 50, key is test_row_0/A:col10/1731708757902/Put/seqid=0 2024-11-15T22:12:37,952 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#A#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:37,952 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/54b44218a1664907babe6a46bc66362e is 50, key is test_row_0/A:col10/1731708757849/Put/seqid=0 2024-11-15T22:12:37,953 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#B#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:37,953 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/2997ea5bdad94de3b761cb06da8c62d0 is 50, key is test_row_0/B:col10/1731708757849/Put/seqid=0 2024-11-15T22:12:37,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742100_1276 (size=12147) 2024-11-15T22:12:37,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742102_1278 (size=12629) 2024-11-15T22:12:37,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742101_1277 (size=12629) 2024-11-15T22:12:38,023 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/54b44218a1664907babe6a46bc66362e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/54b44218a1664907babe6a46bc66362e 2024-11-15T22:12:38,032 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/A of 5d997b3c24773a2e95e850dd849536df into 54b44218a1664907babe6a46bc66362e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:38,032 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:38,032 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/A, priority=13, startTime=1731708757932; duration=0sec 2024-11-15T22:12:38,032 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:38,032 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:A 2024-11-15T22:12:38,032 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:38,034 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:38,034 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/C is initiating minor compaction (all files) 2024-11-15T22:12:38,034 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/C in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:38,034 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/6c9acbbce1094ff0b44fef572f44d04f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/68840900fe6d4a24a6e7447d63951aee, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/12a97b8cf6514472818a2c0cc718aec6] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=36.0 K 2024-11-15T22:12:38,034 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c9acbbce1094ff0b44fef572f44d04f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731708754531 2024-11-15T22:12:38,035 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68840900fe6d4a24a6e7447d63951aee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731708755650 2024-11-15T22:12:38,036 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12a97b8cf6514472818a2c0cc718aec6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731708757849 2024-11-15T22:12:38,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708818050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:38,072 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#C#compaction#227 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:38,073 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/6781176e51e7464ba4057bc3726a9736 is 50, key is test_row_0/C:col10/1731708757849/Put/seqid=0 2024-11-15T22:12:38,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T22:12:38,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742103_1279 (size=12629) 2024-11-15T22:12:38,112 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/6781176e51e7464ba4057bc3726a9736 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/6781176e51e7464ba4057bc3726a9736 2024-11-15T22:12:38,120 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/C of 5d997b3c24773a2e95e850dd849536df into 6781176e51e7464ba4057bc3726a9736(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:38,120 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:38,120 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/C, priority=13, startTime=1731708757933; duration=0sec 2024-11-15T22:12:38,120 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:38,120 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:C 2024-11-15T22:12:38,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:38,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708818166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:38,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:38,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:38,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:38,364 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/c45a2916b4144987a622a479db3c5451 2024-11-15T22:12:38,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/6031500e93a949b99b5b692f489af201 is 50, key is test_row_0/B:col10/1731708757902/Put/seqid=0 2024-11-15T22:12:38,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:38,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708818378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:38,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:38,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:38,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:38,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742104_1280 (size=9757) 2024-11-15T22:12:38,404 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/2997ea5bdad94de3b761cb06da8c62d0 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/2997ea5bdad94de3b761cb06da8c62d0 2024-11-15T22:12:38,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T22:12:38,422 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/B of 5d997b3c24773a2e95e850dd849536df into 2997ea5bdad94de3b761cb06da8c62d0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:38,422 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:38,422 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/B, priority=13, startTime=1731708757932; duration=0sec 2024-11-15T22:12:38,422 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:38,422 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:B 2024-11-15T22:12:38,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708818686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:38,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:38,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:38,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:38,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708818707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:38,709 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_1' in 5d997b3c24773a2e95e850dd849536df of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:38,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:38,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:38,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:38,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708818726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:38,730 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_2' in 5d997b3c24773a2e95e850dd849536df of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:38,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708818726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:38,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708818736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:38,742 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_1' in 5d997b3c24773a2e95e850dd849536df of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:38,743 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_0' in 5d997b3c24773a2e95e850dd849536df of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:38,786 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/6031500e93a949b99b5b692f489af201 2024-11-15T22:12:38,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/d5f7e547901f45379599a692633e4605 is 50, key is test_row_0/C:col10/1731708757902/Put/seqid=0 2024-11-15T22:12:38,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742105_1281 (size=9757) 2024-11-15T22:12:38,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T22:12:39,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708819197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:39,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:39,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:39,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:39,237 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/d5f7e547901f45379599a692633e4605 2024-11-15T22:12:39,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/c45a2916b4144987a622a479db3c5451 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/c45a2916b4144987a622a479db3c5451 2024-11-15T22:12:39,251 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/c45a2916b4144987a622a479db3c5451, entries=150, sequenceid=223, filesize=11.9 K 2024-11-15T22:12:39,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/6031500e93a949b99b5b692f489af201 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/6031500e93a949b99b5b692f489af201 2024-11-15T22:12:39,258 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/6031500e93a949b99b5b692f489af201, entries=100, sequenceid=223, filesize=9.5 K 2024-11-15T22:12:39,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/d5f7e547901f45379599a692633e4605 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/d5f7e547901f45379599a692633e4605 2024-11-15T22:12:39,266 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/d5f7e547901f45379599a692633e4605, entries=100, sequenceid=223, filesize=9.5 K 2024-11-15T22:12:39,267 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 5d997b3c24773a2e95e850dd849536df in 1332ms, sequenceid=223, compaction requested=false 2024-11-15T22:12:39,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:39,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:39,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-15T22:12:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-15T22:12:39,272 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-15T22:12:39,272 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4880 sec 2024-11-15T22:12:39,275 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4950 sec 2024-11-15T22:12:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T22:12:39,937 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:39,937 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:12:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-15T22:12:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T22:12:39,939 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:12:39,940 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:12:39,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:12:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T22:12:40,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T22:12:40,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:40,093 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-15T22:12:40,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/17c183387946428b995aefc7cca7e39a is 50, key is test_row_0/A:col10/1731708758017/Put/seqid=0 2024-11-15T22:12:40,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742106_1282 (size=12151) 2024-11-15T22:12:40,142 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/17c183387946428b995aefc7cca7e39a 2024-11-15T22:12:40,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/0db2126ec7344f7a8ea9e4555537885f is 50, key is test_row_0/B:col10/1731708758017/Put/seqid=0 2024-11-15T22:12:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:40,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:40,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742107_1283 (size=12151) 2024-11-15T22:12:40,225 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/0db2126ec7344f7a8ea9e4555537885f 2024-11-15T22:12:40,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/45f98c1f28b6491b930d9b64dd3ee7a8 is 50, key is test_row_0/C:col10/1731708758017/Put/seqid=0 2024-11-15T22:12:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T22:12:40,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742108_1284 (size=12151) 2024-11-15T22:12:40,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708820305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:40,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:40,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:40,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:40,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:40,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708820417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:40,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:40,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:40,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:40,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T22:12:40,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:40,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708820626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:40,666 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/45f98c1f28b6491b930d9b64dd3ee7a8 2024-11-15T22:12:40,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/17c183387946428b995aefc7cca7e39a as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/17c183387946428b995aefc7cca7e39a 2024-11-15T22:12:40,676 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/17c183387946428b995aefc7cca7e39a, entries=150, sequenceid=246, filesize=11.9 K 2024-11-15T22:12:40,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/0db2126ec7344f7a8ea9e4555537885f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/0db2126ec7344f7a8ea9e4555537885f 2024-11-15T22:12:40,682 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/0db2126ec7344f7a8ea9e4555537885f, entries=150, sequenceid=246, filesize=11.9 K 2024-11-15T22:12:40,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/45f98c1f28b6491b930d9b64dd3ee7a8 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/45f98c1f28b6491b930d9b64dd3ee7a8 2024-11-15T22:12:40,686 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/45f98c1f28b6491b930d9b64dd3ee7a8, entries=150, sequenceid=246, filesize=11.9 K 2024-11-15T22:12:40,688 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 5d997b3c24773a2e95e850dd849536df in 595ms, sequenceid=246, compaction requested=true 2024-11-15T22:12:40,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:40,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:40,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-15T22:12:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-15T22:12:40,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-15T22:12:40,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 749 msec 2024-11-15T22:12:40,693 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 755 msec 2024-11-15T22:12:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:40,941 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:40,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T22:12:40,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/50604b0e26ce414eb8f5b7c31db4f9d3 is 50, key is test_row_0/A:col10/1731708760938/Put/seqid=0 2024-11-15T22:12:40,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742109_1285 (size=12301) 2024-11-15T22:12:41,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708821025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:41,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:41,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:41,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:41,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T22:12:41,076 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:41,077 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:12:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-15T22:12:41,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T22:12:41,079 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:12:41,080 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:12:41,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:12:41,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:41,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708821137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:41,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T22:12:41,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T22:12:41,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:41,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:41,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:41,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:41,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:41,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:41,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708821346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:41,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:41,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:41,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:41,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/50604b0e26ce414eb8f5b7c31db4f9d3 2024-11-15T22:12:41,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T22:12:41,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:41,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:41,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:41,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:41,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:41,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/4f26f83ed4c446f1a29468cebe078490 is 50, key is test_row_0/B:col10/1731708760938/Put/seqid=0 2024-11-15T22:12:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:41,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742110_1286 (size=12301) 2024-11-15T22:12:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T22:12:41,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T22:12:41,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:41,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:41,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:41,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:41,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:41,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:41,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708821656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:41,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:41,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:41,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:41,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T22:12:41,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:41,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:41,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:41,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:41,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:41,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T22:12:41,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/4f26f83ed4c446f1a29468cebe078490 2024-11-15T22:12:41,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/1d8ff77694a84c07a3039e6aa1501867 is 50, key is test_row_0/C:col10/1731708760938/Put/seqid=0 2024-11-15T22:12:41,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742111_1287 (size=12301) 2024-11-15T22:12:41,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T22:12:41,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:41,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:41,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:41,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:41,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:42,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T22:12:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:42,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:42,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:42,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T22:12:42,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:42,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:42,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:42,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:42,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:42,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708822167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:42,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/1d8ff77694a84c07a3039e6aa1501867 2024-11-15T22:12:42,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/50604b0e26ce414eb8f5b7c31db4f9d3 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/50604b0e26ce414eb8f5b7c31db4f9d3 2024-11-15T22:12:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T22:12:42,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/50604b0e26ce414eb8f5b7c31db4f9d3, entries=150, sequenceid=263, filesize=12.0 K 2024-11-15T22:12:42,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/4f26f83ed4c446f1a29468cebe078490 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4f26f83ed4c446f1a29468cebe078490 2024-11-15T22:12:42,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4f26f83ed4c446f1a29468cebe078490, entries=150, sequenceid=263, filesize=12.0 K 2024-11-15T22:12:42,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/1d8ff77694a84c07a3039e6aa1501867 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/1d8ff77694a84c07a3039e6aa1501867 2024-11-15T22:12:42,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/1d8ff77694a84c07a3039e6aa1501867, entries=150, sequenceid=263, filesize=12.0 K 2024-11-15T22:12:42,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 5d997b3c24773a2e95e850dd849536df in 1293ms, sequenceid=263, compaction requested=true 2024-11-15T22:12:42,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:42,234 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:12:42,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:12:42,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:42,235 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:12:42,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:12:42,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:42,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:12:42,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:42,237 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49228 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:12:42,237 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/A is initiating minor compaction (all files) 2024-11-15T22:12:42,237 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/A in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:42,237 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/54b44218a1664907babe6a46bc66362e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/c45a2916b4144987a622a479db3c5451, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/17c183387946428b995aefc7cca7e39a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/50604b0e26ce414eb8f5b7c31db4f9d3] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=48.1 K 2024-11-15T22:12:42,237 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 54b44218a1664907babe6a46bc66362e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731708757849 2024-11-15T22:12:42,237 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46838 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:12:42,238 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/B is initiating minor compaction (all files) 2024-11-15T22:12:42,238 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/B in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:42,238 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/2997ea5bdad94de3b761cb06da8c62d0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/6031500e93a949b99b5b692f489af201, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/0db2126ec7344f7a8ea9e4555537885f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4f26f83ed4c446f1a29468cebe078490] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=45.7 K 2024-11-15T22:12:42,238 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting c45a2916b4144987a622a479db3c5451, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731708757902 2024-11-15T22:12:42,238 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2997ea5bdad94de3b761cb06da8c62d0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731708757849 2024-11-15T22:12:42,239 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 17c183387946428b995aefc7cca7e39a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731708758017 2024-11-15T22:12:42,239 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6031500e93a949b99b5b692f489af201, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731708757902 2024-11-15T22:12:42,239 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 50604b0e26ce414eb8f5b7c31db4f9d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731708760297 2024-11-15T22:12:42,239 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0db2126ec7344f7a8ea9e4555537885f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731708758017 2024-11-15T22:12:42,241 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f26f83ed4c446f1a29468cebe078490, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731708760297 2024-11-15T22:12:42,255 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#A#compaction#236 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:42,256 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/5f025aeaba6b4c4cbca02551dbcc74e4 is 50, key is test_row_0/A:col10/1731708760938/Put/seqid=0 2024-11-15T22:12:42,260 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#B#compaction#237 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:42,260 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/6a8b0e14e4fa4ca7b650c9489da821f4 is 50, key is test_row_0/B:col10/1731708760938/Put/seqid=0 2024-11-15T22:12:42,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T22:12:42,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:42,321 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T22:12:42,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742113_1289 (size=12915) 2024-11-15T22:12:42,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742112_1288 (size=12915) 2024-11-15T22:12:42,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/83fc018ddc4f4fd08b6e1a1174064c7c is 50, key is test_row_0/A:col10/1731708761024/Put/seqid=0 2024-11-15T22:12:42,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742114_1290 (size=12301) 2024-11-15T22:12:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:42,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:42,736 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/6a8b0e14e4fa4ca7b650c9489da821f4 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/6a8b0e14e4fa4ca7b650c9489da821f4 2024-11-15T22:12:42,742 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/B of 5d997b3c24773a2e95e850dd849536df into 6a8b0e14e4fa4ca7b650c9489da821f4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:42,742 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:42,742 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/B, priority=12, startTime=1731708762235; duration=0sec 2024-11-15T22:12:42,742 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:42,742 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:B 2024-11-15T22:12:42,742 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:12:42,745 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/5f025aeaba6b4c4cbca02551dbcc74e4 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5f025aeaba6b4c4cbca02551dbcc74e4 2024-11-15T22:12:42,746 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46838 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:12:42,746 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/C is initiating minor compaction (all files) 2024-11-15T22:12:42,746 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/C in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:42,746 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/6781176e51e7464ba4057bc3726a9736, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/d5f7e547901f45379599a692633e4605, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/45f98c1f28b6491b930d9b64dd3ee7a8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/1d8ff77694a84c07a3039e6aa1501867] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=45.7 K 2024-11-15T22:12:42,746 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6781176e51e7464ba4057bc3726a9736, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731708757849 2024-11-15T22:12:42,747 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5f7e547901f45379599a692633e4605, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731708757902 2024-11-15T22:12:42,749 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45f98c1f28b6491b930d9b64dd3ee7a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731708758017 2024-11-15T22:12:42,750 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d8ff77694a84c07a3039e6aa1501867, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731708760297 2024-11-15T22:12:42,751 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/A of 5d997b3c24773a2e95e850dd849536df into 5f025aeaba6b4c4cbca02551dbcc74e4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:42,751 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:42,751 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/A, priority=12, startTime=1731708762234; duration=0sec 2024-11-15T22:12:42,751 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:42,751 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:A 2024-11-15T22:12:42,755 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/83fc018ddc4f4fd08b6e1a1174064c7c 2024-11-15T22:12:42,768 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#C#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:42,769 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/16e13160bc674843b2869c5eef4264ec is 50, key is test_row_0/C:col10/1731708760938/Put/seqid=0 2024-11-15T22:12:42,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/287f8e2b2fb34e65bf88650f671715ae is 50, key is test_row_0/B:col10/1731708761024/Put/seqid=0 2024-11-15T22:12:42,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742115_1291 (size=12915) 2024-11-15T22:12:42,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:42,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708822777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708822778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:42,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:42,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:42,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:42,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:42,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708822780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:42,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:42,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708822782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:42,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:42,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:42,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:42,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:42,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:42,792 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/16e13160bc674843b2869c5eef4264ec as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/16e13160bc674843b2869c5eef4264ec 2024-11-15T22:12:42,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742116_1292 (size=12301) 2024-11-15T22:12:42,794 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/287f8e2b2fb34e65bf88650f671715ae 2024-11-15T22:12:42,801 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/C of 5d997b3c24773a2e95e850dd849536df into 16e13160bc674843b2869c5eef4264ec(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:42,801 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:42,801 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/C, priority=12, startTime=1731708762235; duration=0sec 2024-11-15T22:12:42,802 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:42,802 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:C 2024-11-15T22:12:42,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/e0a6fcaa4fa447e387db124707200365 is 50, key is test_row_0/C:col10/1731708761024/Put/seqid=0 2024-11-15T22:12:42,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742117_1293 (size=12301) 2024-11-15T22:12:42,824 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/e0a6fcaa4fa447e387db124707200365 2024-11-15T22:12:42,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/83fc018ddc4f4fd08b6e1a1174064c7c as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/83fc018ddc4f4fd08b6e1a1174064c7c 2024-11-15T22:12:42,845 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/83fc018ddc4f4fd08b6e1a1174064c7c, entries=150, sequenceid=282, filesize=12.0 K 2024-11-15T22:12:42,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/287f8e2b2fb34e65bf88650f671715ae as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/287f8e2b2fb34e65bf88650f671715ae 2024-11-15T22:12:42,850 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/287f8e2b2fb34e65bf88650f671715ae, entries=150, sequenceid=282, filesize=12.0 K 2024-11-15T22:12:42,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/e0a6fcaa4fa447e387db124707200365 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/e0a6fcaa4fa447e387db124707200365 2024-11-15T22:12:42,858 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/e0a6fcaa4fa447e387db124707200365, entries=150, sequenceid=282, filesize=12.0 K 2024-11-15T22:12:42,860 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 5d997b3c24773a2e95e850dd849536df in 539ms, sequenceid=282, compaction requested=false 2024-11-15T22:12:42,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:42,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:42,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-15T22:12:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-15T22:12:42,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-15T22:12:42,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7820 sec 2024-11-15T22:12:42,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.7890 sec 2024-11-15T22:12:42,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:42,900 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:42,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-15T22:12:42,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/4d2a1fb87444407b8c306b06d87c9c63 is 50, key is test_row_0/A:col10/1731708762898/Put/seqid=0 2024-11-15T22:12:42,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742118_1294 (size=17181) 2024-11-15T22:12:42,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:42,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:42,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708822917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:42,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708822917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:42,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:42,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:42,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:42,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:42,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:42,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:42,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:42,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708822923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:42,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:42,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:42,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:42,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:42,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708822923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:43,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708823026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:43,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:43,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708823026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:43,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:43,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708823036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:43,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708823036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:43,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:43,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708823187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:43,190 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_0' in 5d997b3c24773a2e95e850dd849536df of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:43,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T22:12:43,226 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:43,227 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:12:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-15T22:12:43,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T22:12:43,230 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:12:43,231 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:12:43,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:12:43,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708823237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:43,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:43,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708823237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:43,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:43,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:43,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708823247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:43,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:43,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708823247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:43,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:43,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:43,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/4d2a1fb87444407b8c306b06d87c9c63 2024-11-15T22:12:43,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/0c29f1f40fe94c90b52676e3936959e0 is 50, key is test_row_0/B:col10/1731708762898/Put/seqid=0 2024-11-15T22:12:43,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742119_1295 (size=12301) 2024-11-15T22:12:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T22:12:43,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T22:12:43,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:43,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:43,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:43,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:43,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:43,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T22:12:43,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T22:12:43,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:43,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:43,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:43,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:43,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:43,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:43,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708823550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:43,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:43,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708823550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:43,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708823556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:43,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:43,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:43,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:43,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708823556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:43,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:43,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:43,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T22:12:43,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:43,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:43,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:43,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:43,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:43,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/0c29f1f40fe94c90b52676e3936959e0 2024-11-15T22:12:43,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/ecd2f7f273f54efea53d6e9d5392322f is 50, key is test_row_0/C:col10/1731708762898/Put/seqid=0 2024-11-15T22:12:43,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742120_1296 (size=12301) 2024-11-15T22:12:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T22:12:43,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T22:12:43,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:43,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:43,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:43,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:43,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:44,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T22:12:44,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:44,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:44,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:44,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:44,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:44,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708824067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:44,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:44,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:44,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:44,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708824067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:44,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:44,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:44,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:44,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708824069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:44,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708824069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:44,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/ecd2f7f273f54efea53d6e9d5392322f 2024-11-15T22:12:44,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/4d2a1fb87444407b8c306b06d87c9c63 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/4d2a1fb87444407b8c306b06d87c9c63 2024-11-15T22:12:44,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/4d2a1fb87444407b8c306b06d87c9c63, entries=250, sequenceid=307, filesize=16.8 K 2024-11-15T22:12:44,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/0c29f1f40fe94c90b52676e3936959e0 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/0c29f1f40fe94c90b52676e3936959e0 2024-11-15T22:12:44,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T22:12:44,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:44,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:44,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:44,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:44,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:44,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/0c29f1f40fe94c90b52676e3936959e0, entries=150, sequenceid=307, filesize=12.0 K 2024-11-15T22:12:44,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/ecd2f7f273f54efea53d6e9d5392322f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/ecd2f7f273f54efea53d6e9d5392322f 2024-11-15T22:12:44,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/ecd2f7f273f54efea53d6e9d5392322f, entries=150, sequenceid=307, filesize=12.0 K 2024-11-15T22:12:44,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 5d997b3c24773a2e95e850dd849536df in 1294ms, sequenceid=307, compaction requested=true 2024-11-15T22:12:44,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:44,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:12:44,194 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:44,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:44,194 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:44,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:12:44,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:44,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:12:44,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:44,197 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:44,197 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/B is initiating minor compaction (all files) 2024-11-15T22:12:44,197 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/B in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:44,197 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/6a8b0e14e4fa4ca7b650c9489da821f4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/287f8e2b2fb34e65bf88650f671715ae, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/0c29f1f40fe94c90b52676e3936959e0] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=36.6 K 2024-11-15T22:12:44,197 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42397 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:44,198 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/A is initiating minor compaction (all files) 2024-11-15T22:12:44,198 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a8b0e14e4fa4ca7b650c9489da821f4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731708760297 2024-11-15T22:12:44,198 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/A in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:44,198 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5f025aeaba6b4c4cbca02551dbcc74e4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/83fc018ddc4f4fd08b6e1a1174064c7c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/4d2a1fb87444407b8c306b06d87c9c63] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=41.4 K 2024-11-15T22:12:44,198 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 287f8e2b2fb34e65bf88650f671715ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731708760997 2024-11-15T22:12:44,198 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f025aeaba6b4c4cbca02551dbcc74e4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731708760297 2024-11-15T22:12:44,199 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83fc018ddc4f4fd08b6e1a1174064c7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731708760997 2024-11-15T22:12:44,199 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c29f1f40fe94c90b52676e3936959e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731708762898 2024-11-15T22:12:44,199 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d2a1fb87444407b8c306b06d87c9c63, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731708762776 2024-11-15T22:12:44,210 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#B#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:44,210 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/9224d27ed05f4aebb757f175b5c503b7 is 50, key is test_row_0/B:col10/1731708762898/Put/seqid=0 2024-11-15T22:12:44,212 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#A#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:44,213 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/e843d03888aa4ff9b3ada7386ac2e4ad is 50, key is test_row_0/A:col10/1731708762898/Put/seqid=0 2024-11-15T22:12:44,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742122_1298 (size=13017) 2024-11-15T22:12:44,226 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/e843d03888aa4ff9b3ada7386ac2e4ad as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/e843d03888aa4ff9b3ada7386ac2e4ad 2024-11-15T22:12:44,232 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/A of 5d997b3c24773a2e95e850dd849536df into e843d03888aa4ff9b3ada7386ac2e4ad(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:44,232 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:44,233 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/A, priority=13, startTime=1731708764194; duration=0sec 2024-11-15T22:12:44,233 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:44,233 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:A 2024-11-15T22:12:44,233 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:44,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742121_1297 (size=13017) 2024-11-15T22:12:44,236 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:44,236 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/C is initiating minor compaction (all files) 2024-11-15T22:12:44,236 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/C in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:44,236 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/16e13160bc674843b2869c5eef4264ec, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/e0a6fcaa4fa447e387db124707200365, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/ecd2f7f273f54efea53d6e9d5392322f] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=36.6 K 2024-11-15T22:12:44,236 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16e13160bc674843b2869c5eef4264ec, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731708760297 2024-11-15T22:12:44,237 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0a6fcaa4fa447e387db124707200365, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731708760997 2024-11-15T22:12:44,237 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecd2f7f273f54efea53d6e9d5392322f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731708762898 2024-11-15T22:12:44,263 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#C#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:44,264 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/7ada511f242748e996d2c4b83360aa66 is 50, key is test_row_0/C:col10/1731708762898/Put/seqid=0 2024-11-15T22:12:44,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742123_1299 (size=13017) 2024-11-15T22:12:44,289 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/7ada511f242748e996d2c4b83360aa66 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/7ada511f242748e996d2c4b83360aa66 2024-11-15T22:12:44,298 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/C of 5d997b3c24773a2e95e850dd849536df into 7ada511f242748e996d2c4b83360aa66(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:44,298 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:44,298 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/C, priority=13, startTime=1731708764194; duration=0sec 2024-11-15T22:12:44,298 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:44,298 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:C 2024-11-15T22:12:44,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T22:12:44,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:44,333 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T22:12:44,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/16ff7f088b274b2696f0baa5680b9e08 is 50, key is test_row_0/A:col10/1731708762921/Put/seqid=0 2024-11-15T22:12:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T22:12:44,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742124_1300 (size=12301) 2024-11-15T22:12:44,651 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/9224d27ed05f4aebb757f175b5c503b7 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/9224d27ed05f4aebb757f175b5c503b7 2024-11-15T22:12:44,658 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/B of 5d997b3c24773a2e95e850dd849536df into 9224d27ed05f4aebb757f175b5c503b7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:44,658 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:44,658 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/B, priority=13, startTime=1731708764194; duration=0sec 2024-11-15T22:12:44,658 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:44,658 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:B 2024-11-15T22:12:44,786 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/16ff7f088b274b2696f0baa5680b9e08 2024-11-15T22:12:44,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/2733212406d2441c8da58064dfd2833f is 50, key is test_row_0/B:col10/1731708762921/Put/seqid=0 2024-11-15T22:12:44,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742125_1301 (size=12301) 2024-11-15T22:12:44,799 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/2733212406d2441c8da58064dfd2833f 2024-11-15T22:12:44,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/1f2b495fd6344e18b3316c12ce549d34 is 50, key is test_row_0/C:col10/1731708762921/Put/seqid=0 2024-11-15T22:12:44,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742126_1302 (size=12301) 2024-11-15T22:12:44,820 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/1f2b495fd6344e18b3316c12ce549d34 2024-11-15T22:12:44,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/16ff7f088b274b2696f0baa5680b9e08 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/16ff7f088b274b2696f0baa5680b9e08 2024-11-15T22:12:44,833 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/16ff7f088b274b2696f0baa5680b9e08, entries=150, sequenceid=323, filesize=12.0 K 2024-11-15T22:12:44,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/2733212406d2441c8da58064dfd2833f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/2733212406d2441c8da58064dfd2833f 2024-11-15T22:12:44,844 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/2733212406d2441c8da58064dfd2833f, entries=150, sequenceid=323, filesize=12.0 K 2024-11-15T22:12:44,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/1f2b495fd6344e18b3316c12ce549d34 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/1f2b495fd6344e18b3316c12ce549d34 2024-11-15T22:12:44,871 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/1f2b495fd6344e18b3316c12ce549d34, entries=150, sequenceid=323, filesize=12.0 K 2024-11-15T22:12:44,872 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for 5d997b3c24773a2e95e850dd849536df in 539ms, sequenceid=323, compaction requested=false 2024-11-15T22:12:44,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:44,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:44,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-15T22:12:44,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-15T22:12:44,876 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-15T22:12:44,876 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6430 sec 2024-11-15T22:12:44,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.6500 sec 2024-11-15T22:12:45,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:45,111 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:45,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T22:12:45,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/60998136b568464c8b63f0e2c8e16977 is 50, key is test_row_0/A:col10/1731708765095/Put/seqid=0 2024-11-15T22:12:45,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742127_1303 (size=12301) 2024-11-15T22:12:45,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708825154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:45,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:45,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:45,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708825157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:45,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:45,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708825159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:45,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:45,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708825163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:45,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:45,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708825208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:45,213 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_0' in 5d997b3c24773a2e95e850dd849536df of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:45,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:45,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:45,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708825267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:45,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708825267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:45,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708825276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:45,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708825276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:45,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T22:12:45,379 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:45,379 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:12:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-15T22:12:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T22:12:45,383 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:12:45,384 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:12:45,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:12:45,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708825476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:45,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708825476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T22:12:45,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708825487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:45,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708825487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:45,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:45,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T22:12:45,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:45,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:45,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:45,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:45,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:45,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/60998136b568464c8b63f0e2c8e16977 2024-11-15T22:12:45,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/df952da08b42444ba61bd8385a713ef4 is 50, key is test_row_0/B:col10/1731708765095/Put/seqid=0 2024-11-15T22:12:45,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742128_1304 (size=12301) 2024-11-15T22:12:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T22:12:45,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T22:12:45,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:45,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:45,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:45,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:45,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:45,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:45,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708825790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:45,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:45,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708825798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:45,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708825801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:45,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708825801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:45,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:45,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:45,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T22:12:45,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:45,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:45,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:45,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:45,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:45,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/df952da08b42444ba61bd8385a713ef4 2024-11-15T22:12:46,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/98bec04c95b24fc284eee421e672abb1 is 50, key is test_row_0/C:col10/1731708765095/Put/seqid=0 2024-11-15T22:12:46,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T22:12:46,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742129_1305 (size=12301) 2024-11-15T22:12:46,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T22:12:46,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:46,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:46,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:46,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:46,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:46,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:46,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T22:12:46,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:46,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:46,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:46,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:46,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:46,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708826305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:46,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708826307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:46,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708826311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:46,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:46,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708826316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:46,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T22:12:46,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:46,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:46,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:46,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:46,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:46,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/98bec04c95b24fc284eee421e672abb1 2024-11-15T22:12:46,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/60998136b568464c8b63f0e2c8e16977 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/60998136b568464c8b63f0e2c8e16977 2024-11-15T22:12:46,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/60998136b568464c8b63f0e2c8e16977, entries=150, sequenceid=336, filesize=12.0 K 2024-11-15T22:12:46,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/df952da08b42444ba61bd8385a713ef4 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/df952da08b42444ba61bd8385a713ef4 2024-11-15T22:12:46,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/df952da08b42444ba61bd8385a713ef4, entries=150, sequenceid=336, filesize=12.0 K 2024-11-15T22:12:46,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/98bec04c95b24fc284eee421e672abb1 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/98bec04c95b24fc284eee421e672abb1 2024-11-15T22:12:46,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T22:12:46,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:46,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:46,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:46,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:46,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:46,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/98bec04c95b24fc284eee421e672abb1, entries=150, sequenceid=336, filesize=12.0 K 2024-11-15T22:12:46,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5d997b3c24773a2e95e850dd849536df in 1392ms, sequenceid=336, compaction requested=true 2024-11-15T22:12:46,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:46,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:12:46,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:46,503 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:46,503 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:46,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:12:46,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:46,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:12:46,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:46,505 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:46,505 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/B is initiating minor compaction (all files) 2024-11-15T22:12:46,505 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/B in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:46,506 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/9224d27ed05f4aebb757f175b5c503b7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/2733212406d2441c8da58064dfd2833f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/df952da08b42444ba61bd8385a713ef4] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=36.7 K 2024-11-15T22:12:46,506 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:46,506 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/A is initiating minor compaction (all files) 2024-11-15T22:12:46,506 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/A in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:46,506 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/e843d03888aa4ff9b3ada7386ac2e4ad, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/16ff7f088b274b2696f0baa5680b9e08, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/60998136b568464c8b63f0e2c8e16977] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=36.7 K 2024-11-15T22:12:46,506 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 9224d27ed05f4aebb757f175b5c503b7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731708762898 2024-11-15T22:12:46,507 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting e843d03888aa4ff9b3ada7386ac2e4ad, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731708762898 2024-11-15T22:12:46,507 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 2733212406d2441c8da58064dfd2833f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731708762906 2024-11-15T22:12:46,507 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16ff7f088b274b2696f0baa5680b9e08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731708762906 2024-11-15T22:12:46,508 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60998136b568464c8b63f0e2c8e16977, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731708765095 2024-11-15T22:12:46,508 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting df952da08b42444ba61bd8385a713ef4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731708765095 2024-11-15T22:12:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T22:12:46,536 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#B#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:46,536 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/cec95333738040369cc6a429a289ac3e is 50, key is test_row_0/B:col10/1731708765095/Put/seqid=0 2024-11-15T22:12:46,548 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#A#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:46,549 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/f933cf4c8e3943a1b692bbbf204873db is 50, key is test_row_0/A:col10/1731708765095/Put/seqid=0 2024-11-15T22:12:46,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742131_1307 (size=13119) 2024-11-15T22:12:46,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742130_1306 (size=13119) 2024-11-15T22:12:46,638 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/cec95333738040369cc6a429a289ac3e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/cec95333738040369cc6a429a289ac3e 2024-11-15T22:12:46,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T22:12:46,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:46,653 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T22:12:46,661 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/B of 5d997b3c24773a2e95e850dd849536df into cec95333738040369cc6a429a289ac3e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:46,661 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:46,662 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/B, priority=13, startTime=1731708766503; duration=0sec 2024-11-15T22:12:46,665 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:46,665 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:B 2024-11-15T22:12:46,666 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:46,669 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:46,669 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/C is initiating minor compaction (all files) 2024-11-15T22:12:46,669 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/C in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:46,669 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/7ada511f242748e996d2c4b83360aa66, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/1f2b495fd6344e18b3316c12ce549d34, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/98bec04c95b24fc284eee421e672abb1] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=36.7 K 2024-11-15T22:12:46,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/5e0c416d60294713a62c720e0cc1a665 is 50, key is test_row_0/A:col10/1731708765155/Put/seqid=0 2024-11-15T22:12:46,673 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ada511f242748e996d2c4b83360aa66, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731708762898 2024-11-15T22:12:46,674 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f2b495fd6344e18b3316c12ce549d34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731708762906 2024-11-15T22:12:46,675 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 98bec04c95b24fc284eee421e672abb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731708765095 2024-11-15T22:12:46,699 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#C#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:46,700 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/a2394b57baa147fa84e10e3a2f1a6a1c is 50, key is test_row_0/C:col10/1731708765095/Put/seqid=0 2024-11-15T22:12:46,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742132_1308 (size=12301) 2024-11-15T22:12:46,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742133_1309 (size=13119) 2024-11-15T22:12:46,789 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/a2394b57baa147fa84e10e3a2f1a6a1c as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/a2394b57baa147fa84e10e3a2f1a6a1c 2024-11-15T22:12:46,810 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/C of 5d997b3c24773a2e95e850dd849536df into a2394b57baa147fa84e10e3a2f1a6a1c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:46,810 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:46,810 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/C, priority=13, startTime=1731708766504; duration=0sec 2024-11-15T22:12:46,812 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:46,812 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:C 2024-11-15T22:12:47,001 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/f933cf4c8e3943a1b692bbbf204873db as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/f933cf4c8e3943a1b692bbbf204873db 2024-11-15T22:12:47,007 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/A of 5d997b3c24773a2e95e850dd849536df into f933cf4c8e3943a1b692bbbf204873db(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:47,008 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:47,008 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/A, priority=13, startTime=1731708766503; duration=0sec 2024-11-15T22:12:47,008 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:47,008 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:A 2024-11-15T22:12:47,118 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/5e0c416d60294713a62c720e0cc1a665 2024-11-15T22:12:47,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/99f4a9a550e64fd6b7f87954d03a5afa is 50, key is test_row_0/B:col10/1731708765155/Put/seqid=0 2024-11-15T22:12:47,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742134_1310 (size=12301) 2024-11-15T22:12:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:47,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:47,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708827341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:47,345 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_2' in 5d997b3c24773a2e95e850dd849536df of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:47,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:47,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:47,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:47,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:47,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708827345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:47,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708827346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:47,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708827348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:47,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:47,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:47,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:47,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708827457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:47,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708827457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:47,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:47,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:47,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:47,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:47,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:47,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:47,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708827466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T22:12:47,580 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/99f4a9a550e64fd6b7f87954d03a5afa 2024-11-15T22:12:47,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/249fe8d760da4ce58d6dc2f72dd15bf2 is 50, key is test_row_0/C:col10/1731708765155/Put/seqid=0 2024-11-15T22:12:47,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742135_1311 (size=12301) 2024-11-15T22:12:47,637 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/249fe8d760da4ce58d6dc2f72dd15bf2 2024-11-15T22:12:47,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/5e0c416d60294713a62c720e0cc1a665 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5e0c416d60294713a62c720e0cc1a665 2024-11-15T22:12:47,653 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5e0c416d60294713a62c720e0cc1a665, entries=150, sequenceid=360, filesize=12.0 K 2024-11-15T22:12:47,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/99f4a9a550e64fd6b7f87954d03a5afa as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/99f4a9a550e64fd6b7f87954d03a5afa 2024-11-15T22:12:47,667 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/99f4a9a550e64fd6b7f87954d03a5afa, entries=150, sequenceid=360, filesize=12.0 K 2024-11-15T22:12:47,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708827666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:47,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:47,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:47,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:47,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/249fe8d760da4ce58d6dc2f72dd15bf2 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/249fe8d760da4ce58d6dc2f72dd15bf2 2024-11-15T22:12:47,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708827670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:47,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708827676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:47,682 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/249fe8d760da4ce58d6dc2f72dd15bf2, entries=150, sequenceid=360, filesize=12.0 K 2024-11-15T22:12:47,683 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 5d997b3c24773a2e95e850dd849536df in 1030ms, sequenceid=360, compaction requested=false 2024-11-15T22:12:47,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:47,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:47,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-15T22:12:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-15T22:12:47,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-15T22:12:47,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3010 sec 2024-11-15T22:12:47,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.3100 sec 2024-11-15T22:12:47,982 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:47,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T22:12:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:48,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/17c755c042ca4adfa05ef175a747102d is 50, key is test_row_0/A:col10/1731708767336/Put/seqid=0 2024-11-15T22:12:48,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742136_1312 (size=12301) 2024-11-15T22:12:48,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:48,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708828036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:48,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708828038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:48,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:48,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708828040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:48,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708828157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:48,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:48,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708828157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:48,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:48,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708828158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:48,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:48,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708828367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:48,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708828367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:48,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708828367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:48,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/17c755c042ca4adfa05ef175a747102d 2024-11-15T22:12:48,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/4c10909ff053413db82ee63ea82f32d9 is 50, key is test_row_0/B:col10/1731708767336/Put/seqid=0 2024-11-15T22:12:48,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742137_1313 (size=12301) 2024-11-15T22:12:48,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/4c10909ff053413db82ee63ea82f32d9 2024-11-15T22:12:48,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/24ee51b7a6de41e8a133c570f1349c1a is 50, key is test_row_0/C:col10/1731708767336/Put/seqid=0 2024-11-15T22:12:48,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742138_1314 (size=12301) 2024-11-15T22:12:48,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/24ee51b7a6de41e8a133c570f1349c1a 2024-11-15T22:12:48,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/17c755c042ca4adfa05ef175a747102d as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/17c755c042ca4adfa05ef175a747102d 2024-11-15T22:12:48,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/17c755c042ca4adfa05ef175a747102d, entries=150, sequenceid=376, filesize=12.0 K 2024-11-15T22:12:48,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/4c10909ff053413db82ee63ea82f32d9 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4c10909ff053413db82ee63ea82f32d9 2024-11-15T22:12:48,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4c10909ff053413db82ee63ea82f32d9, entries=150, sequenceid=376, filesize=12.0 K 2024-11-15T22:12:48,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/24ee51b7a6de41e8a133c570f1349c1a as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/24ee51b7a6de41e8a133c570f1349c1a 2024-11-15T22:12:48,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/24ee51b7a6de41e8a133c570f1349c1a, entries=150, sequenceid=376, filesize=12.0 K 2024-11-15T22:12:48,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 5d997b3c24773a2e95e850dd849536df in 537ms, sequenceid=376, compaction requested=true 2024-11-15T22:12:48,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:48,519 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:48,520 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:48,520 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/A is initiating minor compaction (all files) 2024-11-15T22:12:48,520 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/A in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:48,521 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/f933cf4c8e3943a1b692bbbf204873db, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5e0c416d60294713a62c720e0cc1a665, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/17c755c042ca4adfa05ef175a747102d] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=36.8 K 2024-11-15T22:12:48,522 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting f933cf4c8e3943a1b692bbbf204873db, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731708765095 2024-11-15T22:12:48,522 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e0c416d60294713a62c720e0cc1a665, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731708765155 2024-11-15T22:12:48,523 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17c755c042ca4adfa05ef175a747102d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731708767330 2024-11-15T22:12:48,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:12:48,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:48,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:12:48,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:48,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:12:48,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:12:48,529 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:48,532 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#A#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:48,532 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/5d1c658959364e55aa349a1617938b4f is 50, key is test_row_0/A:col10/1731708767336/Put/seqid=0 2024-11-15T22:12:48,534 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:48,534 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/B is initiating minor compaction (all files) 2024-11-15T22:12:48,534 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/B in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:48,535 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/cec95333738040369cc6a429a289ac3e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/99f4a9a550e64fd6b7f87954d03a5afa, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4c10909ff053413db82ee63ea82f32d9] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=36.8 K 2024-11-15T22:12:48,536 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting cec95333738040369cc6a429a289ac3e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731708765095 2024-11-15T22:12:48,536 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 99f4a9a550e64fd6b7f87954d03a5afa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731708765155 2024-11-15T22:12:48,537 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c10909ff053413db82ee63ea82f32d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731708767330 2024-11-15T22:12:48,558 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#B#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:48,559 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/8d8407c054a8481caff1f3852ab5222a is 50, key is test_row_0/B:col10/1731708767336/Put/seqid=0 2024-11-15T22:12:48,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742139_1315 (size=13221) 2024-11-15T22:12:48,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742140_1316 (size=13221) 2024-11-15T22:12:48,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:48,683 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:48,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T22:12:48,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/da03ae1100354a25b807e70ad9664a35 is 50, key is test_row_0/A:col10/1731708768034/Put/seqid=0 2024-11-15T22:12:48,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742141_1317 (size=12301) 2024-11-15T22:12:48,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708828700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:48,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:48,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708828705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:48,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:48,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708828710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:48,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708828817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:48,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708828817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:48,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:48,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708828817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:48,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:48,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:48,977 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/5d1c658959364e55aa349a1617938b4f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5d1c658959364e55aa349a1617938b4f 2024-11-15T22:12:48,984 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/A of 5d997b3c24773a2e95e850dd849536df into 5d1c658959364e55aa349a1617938b4f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:48,984 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:48,984 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/A, priority=13, startTime=1731708768519; duration=0sec 2024-11-15T22:12:48,984 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:48,984 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:A 2024-11-15T22:12:48,984 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:12:48,986 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:12:48,986 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/C is initiating minor compaction (all files) 2024-11-15T22:12:48,986 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/C in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:48,986 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/a2394b57baa147fa84e10e3a2f1a6a1c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/249fe8d760da4ce58d6dc2f72dd15bf2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/24ee51b7a6de41e8a133c570f1349c1a] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=36.8 K 2024-11-15T22:12:48,987 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2394b57baa147fa84e10e3a2f1a6a1c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731708765095 2024-11-15T22:12:48,987 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 249fe8d760da4ce58d6dc2f72dd15bf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731708765155 2024-11-15T22:12:48,988 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24ee51b7a6de41e8a133c570f1349c1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731708767330 2024-11-15T22:12:48,995 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#C#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:48,997 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/11d9ba5a8d29489ebd6d93dc3b59193b is 50, key is test_row_0/C:col10/1731708767336/Put/seqid=0 2024-11-15T22:12:49,002 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/8d8407c054a8481caff1f3852ab5222a as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/8d8407c054a8481caff1f3852ab5222a 2024-11-15T22:12:49,007 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/B of 5d997b3c24773a2e95e850dd849536df into 8d8407c054a8481caff1f3852ab5222a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:49,007 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:49,007 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/B, priority=13, startTime=1731708768529; duration=0sec 2024-11-15T22:12:49,007 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:49,007 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:B 2024-11-15T22:12:49,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742142_1318 (size=13221) 2024-11-15T22:12:49,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708829027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:49,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708829027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:49,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708829027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:49,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/da03ae1100354a25b807e70ad9664a35 2024-11-15T22:12:49,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/ec03a99e186b4a57849f94498b87f142 is 50, key is test_row_0/B:col10/1731708768034/Put/seqid=0 2024-11-15T22:12:49,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742143_1319 (size=12301) 2024-11-15T22:12:49,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:49,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35106 deadline: 1731708829217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:49,219 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_0' in 5d997b3c24773a2e95e850dd849536df of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:49,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:49,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708829337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:49,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:49,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708829337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:49,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:49,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708829337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:49,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:49,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:49,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:49,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35050 deadline: 1731708829367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:49,369 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_2' in 5d997b3c24773a2e95e850dd849536df of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:12:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:49,429 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/11d9ba5a8d29489ebd6d93dc3b59193b as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/11d9ba5a8d29489ebd6d93dc3b59193b 2024-11-15T22:12:49,438 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/C of 5d997b3c24773a2e95e850dd849536df into 11d9ba5a8d29489ebd6d93dc3b59193b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:49,438 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:49,438 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/C, priority=13, startTime=1731708768529; duration=0sec 2024-11-15T22:12:49,438 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:49,438 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:C 2024-11-15T22:12:49,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/ec03a99e186b4a57849f94498b87f142 2024-11-15T22:12:49,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/c58f2994df434eaf8ffc4a6e042888b6 is 50, key is test_row_0/C:col10/1731708768034/Put/seqid=0 2024-11-15T22:12:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T22:12:49,557 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:49,558 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:12:49,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-15T22:12:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T22:12:49,561 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:12:49,562 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:12:49,562 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:12:49,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742144_1320 (size=12301) 2024-11-15T22:12:49,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/c58f2994df434eaf8ffc4a6e042888b6 2024-11-15T22:12:49,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/da03ae1100354a25b807e70ad9664a35 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/da03ae1100354a25b807e70ad9664a35 2024-11-15T22:12:49,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/da03ae1100354a25b807e70ad9664a35, entries=150, sequenceid=402, filesize=12.0 K 2024-11-15T22:12:49,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/ec03a99e186b4a57849f94498b87f142 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/ec03a99e186b4a57849f94498b87f142 2024-11-15T22:12:49,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/ec03a99e186b4a57849f94498b87f142, entries=150, sequenceid=402, filesize=12.0 K 2024-11-15T22:12:49,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/c58f2994df434eaf8ffc4a6e042888b6 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/c58f2994df434eaf8ffc4a6e042888b6 2024-11-15T22:12:49,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/c58f2994df434eaf8ffc4a6e042888b6, entries=150, sequenceid=402, filesize=12.0 K 2024-11-15T22:12:49,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 5d997b3c24773a2e95e850dd849536df in 911ms, sequenceid=402, compaction requested=false 2024-11-15T22:12:49,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T22:12:49,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T22:12:49,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:49,715 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-15T22:12:49,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/5ae0fcdd34764561bb1bb42c21235717 is 50, key is test_row_0/A:col10/1731708768693/Put/seqid=0 2024-11-15T22:12:49,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742145_1321 (size=12301) 2024-11-15T22:12:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:49,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T22:12:49,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:49,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708829898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:49,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:49,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708829907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:49,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708829907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:49,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:50,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708830017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:50,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708830017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:50,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708830017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:50,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:50,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:50,131 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/5ae0fcdd34764561bb1bb42c21235717 2024-11-15T22:12:50,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/11b72edf0ab5407297be4b0320980810 is 50, key is test_row_0/B:col10/1731708768693/Put/seqid=0 2024-11-15T22:12:50,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742146_1322 (size=12301) 2024-11-15T22:12:50,156 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/11b72edf0ab5407297be4b0320980810 2024-11-15T22:12:50,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/a7d2df106a87461581afbebb65bcaf2c is 50, key is test_row_0/C:col10/1731708768693/Put/seqid=0 2024-11-15T22:12:50,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T22:12:50,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742147_1323 (size=12301) 2024-11-15T22:12:50,192 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/a7d2df106a87461581afbebb65bcaf2c 2024-11-15T22:12:50,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/5ae0fcdd34764561bb1bb42c21235717 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5ae0fcdd34764561bb1bb42c21235717 2024-11-15T22:12:50,205 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5ae0fcdd34764561bb1bb42c21235717, entries=150, sequenceid=415, filesize=12.0 K 2024-11-15T22:12:50,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/11b72edf0ab5407297be4b0320980810 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/11b72edf0ab5407297be4b0320980810 2024-11-15T22:12:50,213 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/11b72edf0ab5407297be4b0320980810, entries=150, sequenceid=415, filesize=12.0 K 2024-11-15T22:12:50,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/a7d2df106a87461581afbebb65bcaf2c as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/a7d2df106a87461581afbebb65bcaf2c 2024-11-15T22:12:50,221 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/a7d2df106a87461581afbebb65bcaf2c, entries=150, sequenceid=415, filesize=12.0 K 2024-11-15T22:12:50,222 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 5d997b3c24773a2e95e850dd849536df in 507ms, sequenceid=415, compaction requested=true 2024-11-15T22:12:50,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:50,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:50,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-15T22:12:50,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-15T22:12:50,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-15T22:12:50,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 662 msec 2024-11-15T22:12:50,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 669 msec 2024-11-15T22:12:50,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:50,233 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:12:50,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-15T22:12:50,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/18c60bf0d2b04ed1bf9ea3795b908a1b is 50, key is test_row_0/A:col10/1731708770232/Put/seqid=0 2024-11-15T22:12:50,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:50,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708830241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:50,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:50,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:50,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708830247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:50,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:50,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708830248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:50,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742148_1324 (size=17181) 2024-11-15T22:12:50,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:50,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708830365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:50,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:50,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708830365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:50,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:50,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708830367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:50,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:50,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708830576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:50,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:50,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708830576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:50,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708830586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:50,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:50,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/18c60bf0d2b04ed1bf9ea3795b908a1b 2024-11-15T22:12:50,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T22:12:50,697 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:50,698 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:12:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-15T22:12:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T22:12:50,707 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:12:50,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/a46db65cfe4d44408b49a717923c1bde is 50, key is test_row_0/B:col10/1731708770232/Put/seqid=0 2024-11-15T22:12:50,709 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:12:50,709 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:12:50,713 INFO [Thread-1108 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1108. 2024-11-15T22:12:50,714 DEBUG [Thread-1108 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:12:50,714 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:50,714 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:50,714 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:12:50,714 INFO [Thread-1110 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1110. 2024-11-15T22:12:50,714 DEBUG [Thread-1110 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:12:50,714 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:50,715 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:50,715 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:12:50,715 INFO [Thread-1106 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1106. 2024-11-15T22:12:50,715 DEBUG [Thread-1106 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:12:50,715 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:50,715 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:50,715 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:12:50,718 INFO [Thread-1112 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1112. 2024-11-15T22:12:50,718 DEBUG [Thread-1112 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:12:50,718 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:50,718 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:50,718 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:12:50,718 INFO [Thread-1104 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1104. 2024-11-15T22:12:50,718 DEBUG [Thread-1104 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:12:50,718 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:50,718 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:50,718 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:12:50,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742149_1325 (size=12301) 2024-11-15T22:12:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T22:12:50,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T22:12:50,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:50,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:50,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:50,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:50,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:50,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:50,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35128 deadline: 1731708830887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35078 deadline: 1731708830887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:50,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:12:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35044 deadline: 1731708830896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:12:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:12:51,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T22:12:51,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:51,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:51,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:51,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:51,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T22:12:51,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/a46db65cfe4d44408b49a717923c1bde 2024-11-15T22:12:51,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/24d4cd96a54946b2818f26eaf11750d6 is 50, key is test_row_0/C:col10/1731708770232/Put/seqid=0 2024-11-15T22:12:51,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742150_1326 (size=12301) 2024-11-15T22:12:51,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/24d4cd96a54946b2818f26eaf11750d6 2024-11-15T22:12:51,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/18c60bf0d2b04ed1bf9ea3795b908a1b as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/18c60bf0d2b04ed1bf9ea3795b908a1b 2024-11-15T22:12:51,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T22:12:51,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:51,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. as already flushing 2024-11-15T22:12:51,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:51,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:51,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:12:51,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/18c60bf0d2b04ed1bf9ea3795b908a1b, entries=250, sequenceid=445, filesize=16.8 K 2024-11-15T22:12:51,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/a46db65cfe4d44408b49a717923c1bde as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/a46db65cfe4d44408b49a717923c1bde 2024-11-15T22:12:51,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/a46db65cfe4d44408b49a717923c1bde, entries=150, sequenceid=445, filesize=12.0 K 2024-11-15T22:12:51,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/24d4cd96a54946b2818f26eaf11750d6 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/24d4cd96a54946b2818f26eaf11750d6 2024-11-15T22:12:51,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/24d4cd96a54946b2818f26eaf11750d6, entries=150, sequenceid=445, filesize=12.0 K 2024-11-15T22:12:51,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 5d997b3c24773a2e95e850dd849536df in 1031ms, sequenceid=445, compaction requested=true 2024-11-15T22:12:51,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:51,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:12:51,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:51,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:12:51,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:12:51,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5d997b3c24773a2e95e850dd849536df:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:12:51,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:12:51,265 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:12:51,266 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:12:51,283 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:12:51,283 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/A is initiating minor compaction (all files) 2024-11-15T22:12:51,283 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/A in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:51,283 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5d1c658959364e55aa349a1617938b4f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/da03ae1100354a25b807e70ad9664a35, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5ae0fcdd34764561bb1bb42c21235717, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/18c60bf0d2b04ed1bf9ea3795b908a1b] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=53.7 K 2024-11-15T22:12:51,285 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:12:51,285 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/C is initiating minor compaction (all files) 2024-11-15T22:12:51,285 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/C in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:51,285 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/11d9ba5a8d29489ebd6d93dc3b59193b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/c58f2994df434eaf8ffc4a6e042888b6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/a7d2df106a87461581afbebb65bcaf2c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/24d4cd96a54946b2818f26eaf11750d6] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=48.9 K 2024-11-15T22:12:51,297 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d1c658959364e55aa349a1617938b4f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731708767330 2024-11-15T22:12:51,305 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11d9ba5a8d29489ebd6d93dc3b59193b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731708767330 2024-11-15T22:12:51,308 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting da03ae1100354a25b807e70ad9664a35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731708768034 2024-11-15T22:12:51,313 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting c58f2994df434eaf8ffc4a6e042888b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731708768034 2024-11-15T22:12:51,317 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ae0fcdd34764561bb1bb42c21235717, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731708768693 2024-11-15T22:12:51,325 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 18c60bf0d2b04ed1bf9ea3795b908a1b, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731708769901 2024-11-15T22:12:51,325 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7d2df106a87461581afbebb65bcaf2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731708768693 2024-11-15T22:12:51,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T22:12:51,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:51,326 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-15T22:12:51,333 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24d4cd96a54946b2818f26eaf11750d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731708769904 2024-11-15T22:12:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T22:12:51,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/7cb0b40db3e148ceae54361463790e17 is 50, key is test_row_0/A:col10/1731708770239/Put/seqid=0 2024-11-15T22:12:51,364 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#C#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:51,365 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/9dff5c3b5c944b88837f11f3b3598b6e is 50, key is test_row_0/C:col10/1731708770232/Put/seqid=0 2024-11-15T22:12:51,385 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#A#compaction#277 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:51,390 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/462ef0f73c1e452d99b5581e442479b3 is 50, key is test_row_0/A:col10/1731708770232/Put/seqid=0 2024-11-15T22:12:51,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742151_1327 (size=12301) 2024-11-15T22:12:51,400 INFO [Thread-1101 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1101. 2024-11-15T22:12:51,400 DEBUG [Thread-1101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:12:51,400 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:51,400 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:51,400 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:12:51,402 INFO [Thread-1099 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1099. 2024-11-15T22:12:51,402 DEBUG [Thread-1099 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:12:51,402 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:51,402 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:51,402 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:12:51,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742152_1328 (size=13357) 2024-11-15T22:12:51,412 INFO [Thread-1093 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1093. 2024-11-15T22:12:51,412 DEBUG [Thread-1093 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:12:51,412 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:51,412 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/9dff5c3b5c944b88837f11f3b3598b6e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/9dff5c3b5c944b88837f11f3b3598b6e 2024-11-15T22:12:51,412 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:51,412 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:12:51,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742153_1329 (size=13357) 2024-11-15T22:12:51,426 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/C of 5d997b3c24773a2e95e850dd849536df into 9dff5c3b5c944b88837f11f3b3598b6e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:51,426 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:51,426 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/C, priority=12, startTime=1731708771265; duration=0sec 2024-11-15T22:12:51,426 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:12:51,427 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:C 2024-11-15T22:12:51,427 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:12:51,429 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:12:51,430 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 5d997b3c24773a2e95e850dd849536df/B is initiating minor compaction (all files) 2024-11-15T22:12:51,430 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5d997b3c24773a2e95e850dd849536df/B in TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:51,430 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/462ef0f73c1e452d99b5581e442479b3 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/462ef0f73c1e452d99b5581e442479b3 2024-11-15T22:12:51,430 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/8d8407c054a8481caff1f3852ab5222a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/ec03a99e186b4a57849f94498b87f142, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/11b72edf0ab5407297be4b0320980810, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/a46db65cfe4d44408b49a717923c1bde] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp, totalSize=48.9 K 2024-11-15T22:12:51,431 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d8407c054a8481caff1f3852ab5222a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731708767330 2024-11-15T22:12:51,432 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec03a99e186b4a57849f94498b87f142, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731708768034 2024-11-15T22:12:51,432 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11b72edf0ab5407297be4b0320980810, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731708768693 2024-11-15T22:12:51,433 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting a46db65cfe4d44408b49a717923c1bde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731708769904 2024-11-15T22:12:51,436 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/A of 5d997b3c24773a2e95e850dd849536df into 462ef0f73c1e452d99b5581e442479b3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:51,436 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:51,436 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/A, priority=12, startTime=1731708771265; duration=0sec 2024-11-15T22:12:51,436 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:51,437 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:A 2024-11-15T22:12:51,450 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5d997b3c24773a2e95e850dd849536df#B#compaction#278 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:12:51,451 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/550968a7f55b44f8a808bed67a81e2d8 is 50, key is test_row_0/B:col10/1731708770232/Put/seqid=0 2024-11-15T22:12:51,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742154_1330 (size=13357) 2024-11-15T22:12:51,471 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/550968a7f55b44f8a808bed67a81e2d8 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/550968a7f55b44f8a808bed67a81e2d8 2024-11-15T22:12:51,476 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5d997b3c24773a2e95e850dd849536df/B of 5d997b3c24773a2e95e850dd849536df into 550968a7f55b44f8a808bed67a81e2d8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:12:51,476 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:51,476 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., storeName=5d997b3c24773a2e95e850dd849536df/B, priority=12, startTime=1731708771265; duration=0sec 2024-11-15T22:12:51,476 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:12:51,476 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5d997b3c24773a2e95e850dd849536df:B 2024-11-15T22:12:51,791 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/7cb0b40db3e148ceae54361463790e17 2024-11-15T22:12:51,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/4c3d2faa82a043fa90d82f3a41bbd89b is 50, key is test_row_0/B:col10/1731708770239/Put/seqid=0 2024-11-15T22:12:51,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742155_1331 (size=12301) 2024-11-15T22:12:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T22:12:52,203 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/4c3d2faa82a043fa90d82f3a41bbd89b 2024-11-15T22:12:52,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/985ea7a76b884eedb7cb2cce57ed2063 is 50, key is test_row_0/C:col10/1731708770239/Put/seqid=0 2024-11-15T22:12:52,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742156_1332 (size=12301) 2024-11-15T22:12:52,614 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/985ea7a76b884eedb7cb2cce57ed2063 2024-11-15T22:12:52,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/7cb0b40db3e148ceae54361463790e17 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/7cb0b40db3e148ceae54361463790e17 2024-11-15T22:12:52,623 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/7cb0b40db3e148ceae54361463790e17, entries=150, sequenceid=451, filesize=12.0 K 2024-11-15T22:12:52,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/4c3d2faa82a043fa90d82f3a41bbd89b as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4c3d2faa82a043fa90d82f3a41bbd89b 2024-11-15T22:12:52,627 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4c3d2faa82a043fa90d82f3a41bbd89b, entries=150, sequenceid=451, filesize=12.0 K 2024-11-15T22:12:52,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/985ea7a76b884eedb7cb2cce57ed2063 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/985ea7a76b884eedb7cb2cce57ed2063 2024-11-15T22:12:52,631 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/985ea7a76b884eedb7cb2cce57ed2063, entries=150, sequenceid=451, filesize=12.0 K 2024-11-15T22:12:52,632 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=20.13 KB/20610 for 5d997b3c24773a2e95e850dd849536df in 1306ms, sequenceid=451, compaction requested=false 2024-11-15T22:12:52,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 5d997b3c24773a2e95e850dd849536df: 2024-11-15T22:12:52,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:52,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-15T22:12:52,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-15T22:12:52,636 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-15T22:12:52,636 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9250 sec 2024-11-15T22:12:52,638 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.9390 sec 2024-11-15T22:12:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T22:12:52,857 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:12:53,418 INFO [Thread-1095 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1095. 2024-11-15T22:12:53,418 DEBUG [Thread-1095 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:12:53,418 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:53,419 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:53,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:12:54,574 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T22:12:59,260 INFO [Thread-1097 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1097. 2024-11-15T22:12:59,260 DEBUG [Thread-1097 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:12:59,260 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:59,260 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:59,260 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:12:59,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T22:12:59,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-15T22:12:59,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-15T22:12:59,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-15T22:12:59,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-15T22:12:59,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-15T22:12:59,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T22:12:59,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3128 2024-11-15T22:12:59,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3184 2024-11-15T22:12:59,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3160 2024-11-15T22:12:59,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3141 2024-11-15T22:12:59,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3245 2024-11-15T22:12:59,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T22:12:59,261 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T22:12:59,261 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:12:59,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:59,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:12:59,261 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:12:59,262 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T22:12:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T22:12:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T22:12:59,266 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708779266"}]},"ts":"1731708779266"} 2024-11-15T22:12:59,268 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T22:12:59,268 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T22:12:59,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T22:12:59,270 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d997b3c24773a2e95e850dd849536df, UNASSIGN}] 2024-11-15T22:12:59,271 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d997b3c24773a2e95e850dd849536df, UNASSIGN 2024-11-15T22:12:59,272 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=5d997b3c24773a2e95e850dd849536df, regionState=CLOSING, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:12:59,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d997b3c24773a2e95e850dd849536df, UNASSIGN because future has completed 2024-11-15T22:12:59,273 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T22:12:59,273 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516}] 2024-11-15T22:12:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T22:12:59,426 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:12:59,426 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T22:12:59,426 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 5d997b3c24773a2e95e850dd849536df, disabling compactions & flushes 2024-11-15T22:12:59,426 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:59,426 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:59,426 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. after waiting 0 ms 2024-11-15T22:12:59,426 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:12:59,426 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing 5d997b3c24773a2e95e850dd849536df 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T22:12:59,430 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/f78b88f5c657488180fb9c2030a439b5 is 50, key is test_row_0/A:col10/1731708779258/Put/seqid=0 2024-11-15T22:12:59,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742157_1333 (size=12301) 2024-11-15T22:12:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T22:12:59,835 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/f78b88f5c657488180fb9c2030a439b5 2024-11-15T22:12:59,842 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/dcead682f55c41139df133e627fae895 is 50, key is test_row_0/B:col10/1731708779258/Put/seqid=0 2024-11-15T22:12:59,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742158_1334 (size=12301) 2024-11-15T22:12:59,845 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/dcead682f55c41139df133e627fae895 2024-11-15T22:12:59,851 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/27666c1ffcfc47d29504bd9fb8205035 is 50, key is test_row_0/C:col10/1731708779258/Put/seqid=0 2024-11-15T22:12:59,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742159_1335 (size=12301) 2024-11-15T22:12:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T22:13:00,255 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/27666c1ffcfc47d29504bd9fb8205035 2024-11-15T22:13:00,260 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/A/f78b88f5c657488180fb9c2030a439b5 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/f78b88f5c657488180fb9c2030a439b5 2024-11-15T22:13:00,263 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/f78b88f5c657488180fb9c2030a439b5, entries=150, sequenceid=462, filesize=12.0 K 2024-11-15T22:13:00,264 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/B/dcead682f55c41139df133e627fae895 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/dcead682f55c41139df133e627fae895 2024-11-15T22:13:00,268 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/dcead682f55c41139df133e627fae895, entries=150, sequenceid=462, filesize=12.0 K 2024-11-15T22:13:00,268 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/.tmp/C/27666c1ffcfc47d29504bd9fb8205035 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/27666c1ffcfc47d29504bd9fb8205035 2024-11-15T22:13:00,272 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/27666c1ffcfc47d29504bd9fb8205035, entries=150, sequenceid=462, filesize=12.0 K 2024-11-15T22:13:00,273 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 5d997b3c24773a2e95e850dd849536df in 847ms, sequenceid=462, compaction requested=true 2024-11-15T22:13:00,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/0e4d3f74131b44679ba1545f5e7e6bd2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/49cbcb1b5d75415c86f1d373f22a6ef4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/75932caeea48404088088c3f93189515, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/723b7bcb1e294788a71116aa7093a82d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/0b64a02ef2164d05b9cd4a8877bdba95, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/14208973c8e54e39a53309f612071651, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/a468bf814c5946c890fc01e546bf3e79, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/c60c5b1a253047e28cbe6608f34e36a2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/a352c4a0854b43a79f61324a7054f0e8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/bde33d909ea049c4b55b8a608db8f2ed, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/7673ee6f673d45b288eb81f73296d3f6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/dcc963b6d386421e833002af4a7c68e1, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/963e7989ff5e4fdf94c306eed846ff04, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/06578bf97e2f4726813df336bd5bfa6c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/54b44218a1664907babe6a46bc66362e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/c45a2916b4144987a622a479db3c5451, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/17c183387946428b995aefc7cca7e39a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5f025aeaba6b4c4cbca02551dbcc74e4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/50604b0e26ce414eb8f5b7c31db4f9d3, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/83fc018ddc4f4fd08b6e1a1174064c7c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/4d2a1fb87444407b8c306b06d87c9c63, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/e843d03888aa4ff9b3ada7386ac2e4ad, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/16ff7f088b274b2696f0baa5680b9e08, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/f933cf4c8e3943a1b692bbbf204873db, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/60998136b568464c8b63f0e2c8e16977, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5e0c416d60294713a62c720e0cc1a665, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5d1c658959364e55aa349a1617938b4f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/17c755c042ca4adfa05ef175a747102d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/da03ae1100354a25b807e70ad9664a35, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5ae0fcdd34764561bb1bb42c21235717, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/18c60bf0d2b04ed1bf9ea3795b908a1b] to archive 2024-11-15T22:13:00,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:13:00,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/0e4d3f74131b44679ba1545f5e7e6bd2 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/0e4d3f74131b44679ba1545f5e7e6bd2 2024-11-15T22:13:00,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/49cbcb1b5d75415c86f1d373f22a6ef4 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/49cbcb1b5d75415c86f1d373f22a6ef4 2024-11-15T22:13:00,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/75932caeea48404088088c3f93189515 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/75932caeea48404088088c3f93189515 2024-11-15T22:13:00,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/723b7bcb1e294788a71116aa7093a82d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/723b7bcb1e294788a71116aa7093a82d 2024-11-15T22:13:00,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/0b64a02ef2164d05b9cd4a8877bdba95 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/0b64a02ef2164d05b9cd4a8877bdba95 2024-11-15T22:13:00,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/14208973c8e54e39a53309f612071651 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/14208973c8e54e39a53309f612071651 2024-11-15T22:13:00,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/a468bf814c5946c890fc01e546bf3e79 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/a468bf814c5946c890fc01e546bf3e79 2024-11-15T22:13:00,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/c60c5b1a253047e28cbe6608f34e36a2 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/c60c5b1a253047e28cbe6608f34e36a2 2024-11-15T22:13:00,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/a352c4a0854b43a79f61324a7054f0e8 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/a352c4a0854b43a79f61324a7054f0e8 2024-11-15T22:13:00,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/bde33d909ea049c4b55b8a608db8f2ed to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/bde33d909ea049c4b55b8a608db8f2ed 2024-11-15T22:13:00,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/7673ee6f673d45b288eb81f73296d3f6 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/7673ee6f673d45b288eb81f73296d3f6 2024-11-15T22:13:00,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/dcc963b6d386421e833002af4a7c68e1 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/dcc963b6d386421e833002af4a7c68e1 2024-11-15T22:13:00,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/963e7989ff5e4fdf94c306eed846ff04 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/963e7989ff5e4fdf94c306eed846ff04 2024-11-15T22:13:00,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/06578bf97e2f4726813df336bd5bfa6c to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/06578bf97e2f4726813df336bd5bfa6c 2024-11-15T22:13:00,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/54b44218a1664907babe6a46bc66362e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/54b44218a1664907babe6a46bc66362e 2024-11-15T22:13:00,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/c45a2916b4144987a622a479db3c5451 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/c45a2916b4144987a622a479db3c5451 2024-11-15T22:13:00,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/17c183387946428b995aefc7cca7e39a to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/17c183387946428b995aefc7cca7e39a 2024-11-15T22:13:00,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5f025aeaba6b4c4cbca02551dbcc74e4 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5f025aeaba6b4c4cbca02551dbcc74e4 2024-11-15T22:13:00,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/50604b0e26ce414eb8f5b7c31db4f9d3 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/50604b0e26ce414eb8f5b7c31db4f9d3 2024-11-15T22:13:00,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/83fc018ddc4f4fd08b6e1a1174064c7c to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/83fc018ddc4f4fd08b6e1a1174064c7c 2024-11-15T22:13:00,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/4d2a1fb87444407b8c306b06d87c9c63 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/4d2a1fb87444407b8c306b06d87c9c63 2024-11-15T22:13:00,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/e843d03888aa4ff9b3ada7386ac2e4ad to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/e843d03888aa4ff9b3ada7386ac2e4ad 2024-11-15T22:13:00,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/16ff7f088b274b2696f0baa5680b9e08 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/16ff7f088b274b2696f0baa5680b9e08 2024-11-15T22:13:00,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/f933cf4c8e3943a1b692bbbf204873db to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/f933cf4c8e3943a1b692bbbf204873db 2024-11-15T22:13:00,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/60998136b568464c8b63f0e2c8e16977 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/60998136b568464c8b63f0e2c8e16977 2024-11-15T22:13:00,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5e0c416d60294713a62c720e0cc1a665 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5e0c416d60294713a62c720e0cc1a665 2024-11-15T22:13:00,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5d1c658959364e55aa349a1617938b4f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5d1c658959364e55aa349a1617938b4f 2024-11-15T22:13:00,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/17c755c042ca4adfa05ef175a747102d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/17c755c042ca4adfa05ef175a747102d 2024-11-15T22:13:00,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/da03ae1100354a25b807e70ad9664a35 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/da03ae1100354a25b807e70ad9664a35 2024-11-15T22:13:00,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5ae0fcdd34764561bb1bb42c21235717 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/5ae0fcdd34764561bb1bb42c21235717 2024-11-15T22:13:00,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/18c60bf0d2b04ed1bf9ea3795b908a1b to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/18c60bf0d2b04ed1bf9ea3795b908a1b 2024-11-15T22:13:00,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/515162c9010948bcac3123650d996bd2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/0a752d46426c49babcac33d8ece50ada, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/6d0925f6ecd846a0ab897a92b0305298, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/dac37fe4251e47e7a75ad336f116c2ef, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/3d03efb84c604383a38547bbd9783c6e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/1b16e3b3f6264e4bb62b3232f242ee61, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/92ffb9df02ea4ac7b17ec1cdfe4892f2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/40836c91f99d465ab1edc2a987974ef3, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/d1b6a592bc074b01a28c817bf6e734da, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/b265d3fbb04341c2aa56668f11d0437a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/752b6c46d0194e508044122687e4c56f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4e8e8fc6250649c6ad0dce3827faf4a0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/7c6fae07d4074fc7a88649f26a0c3d46, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/2997ea5bdad94de3b761cb06da8c62d0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/177b842d6ab94367af98a89b7debb4a7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/6031500e93a949b99b5b692f489af201, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/0db2126ec7344f7a8ea9e4555537885f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/6a8b0e14e4fa4ca7b650c9489da821f4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4f26f83ed4c446f1a29468cebe078490, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/287f8e2b2fb34e65bf88650f671715ae, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/9224d27ed05f4aebb757f175b5c503b7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/0c29f1f40fe94c90b52676e3936959e0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/2733212406d2441c8da58064dfd2833f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/cec95333738040369cc6a429a289ac3e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/df952da08b42444ba61bd8385a713ef4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/99f4a9a550e64fd6b7f87954d03a5afa, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/8d8407c054a8481caff1f3852ab5222a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4c10909ff053413db82ee63ea82f32d9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/ec03a99e186b4a57849f94498b87f142, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/11b72edf0ab5407297be4b0320980810, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/a46db65cfe4d44408b49a717923c1bde] to archive 2024-11-15T22:13:00,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:13:00,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/515162c9010948bcac3123650d996bd2 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/515162c9010948bcac3123650d996bd2 2024-11-15T22:13:00,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/0a752d46426c49babcac33d8ece50ada to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/0a752d46426c49babcac33d8ece50ada 2024-11-15T22:13:00,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/6d0925f6ecd846a0ab897a92b0305298 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/6d0925f6ecd846a0ab897a92b0305298 2024-11-15T22:13:00,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/dac37fe4251e47e7a75ad336f116c2ef to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/dac37fe4251e47e7a75ad336f116c2ef 2024-11-15T22:13:00,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/3d03efb84c604383a38547bbd9783c6e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/3d03efb84c604383a38547bbd9783c6e 2024-11-15T22:13:00,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/1b16e3b3f6264e4bb62b3232f242ee61 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/1b16e3b3f6264e4bb62b3232f242ee61 2024-11-15T22:13:00,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/92ffb9df02ea4ac7b17ec1cdfe4892f2 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/92ffb9df02ea4ac7b17ec1cdfe4892f2 2024-11-15T22:13:00,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/40836c91f99d465ab1edc2a987974ef3 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/40836c91f99d465ab1edc2a987974ef3 2024-11-15T22:13:00,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/d1b6a592bc074b01a28c817bf6e734da to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/d1b6a592bc074b01a28c817bf6e734da 2024-11-15T22:13:00,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/b265d3fbb04341c2aa56668f11d0437a to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/b265d3fbb04341c2aa56668f11d0437a 2024-11-15T22:13:00,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/752b6c46d0194e508044122687e4c56f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/752b6c46d0194e508044122687e4c56f 2024-11-15T22:13:00,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4e8e8fc6250649c6ad0dce3827faf4a0 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4e8e8fc6250649c6ad0dce3827faf4a0 2024-11-15T22:13:00,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/7c6fae07d4074fc7a88649f26a0c3d46 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/7c6fae07d4074fc7a88649f26a0c3d46 2024-11-15T22:13:00,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/2997ea5bdad94de3b761cb06da8c62d0 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/2997ea5bdad94de3b761cb06da8c62d0 2024-11-15T22:13:00,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/177b842d6ab94367af98a89b7debb4a7 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/177b842d6ab94367af98a89b7debb4a7 2024-11-15T22:13:00,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/6031500e93a949b99b5b692f489af201 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/6031500e93a949b99b5b692f489af201 2024-11-15T22:13:00,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/0db2126ec7344f7a8ea9e4555537885f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/0db2126ec7344f7a8ea9e4555537885f 2024-11-15T22:13:00,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/6a8b0e14e4fa4ca7b650c9489da821f4 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/6a8b0e14e4fa4ca7b650c9489da821f4 2024-11-15T22:13:00,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4f26f83ed4c446f1a29468cebe078490 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4f26f83ed4c446f1a29468cebe078490 2024-11-15T22:13:00,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/287f8e2b2fb34e65bf88650f671715ae to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/287f8e2b2fb34e65bf88650f671715ae 2024-11-15T22:13:00,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/9224d27ed05f4aebb757f175b5c503b7 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/9224d27ed05f4aebb757f175b5c503b7 2024-11-15T22:13:00,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/0c29f1f40fe94c90b52676e3936959e0 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/0c29f1f40fe94c90b52676e3936959e0 2024-11-15T22:13:00,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/2733212406d2441c8da58064dfd2833f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/2733212406d2441c8da58064dfd2833f 2024-11-15T22:13:00,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/cec95333738040369cc6a429a289ac3e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/cec95333738040369cc6a429a289ac3e 2024-11-15T22:13:00,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/df952da08b42444ba61bd8385a713ef4 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/df952da08b42444ba61bd8385a713ef4 2024-11-15T22:13:00,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/99f4a9a550e64fd6b7f87954d03a5afa to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/99f4a9a550e64fd6b7f87954d03a5afa 2024-11-15T22:13:00,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/8d8407c054a8481caff1f3852ab5222a to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/8d8407c054a8481caff1f3852ab5222a 2024-11-15T22:13:00,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4c10909ff053413db82ee63ea82f32d9 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4c10909ff053413db82ee63ea82f32d9 2024-11-15T22:13:00,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/ec03a99e186b4a57849f94498b87f142 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/ec03a99e186b4a57849f94498b87f142 2024-11-15T22:13:00,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/11b72edf0ab5407297be4b0320980810 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/11b72edf0ab5407297be4b0320980810 2024-11-15T22:13:00,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/a46db65cfe4d44408b49a717923c1bde to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/a46db65cfe4d44408b49a717923c1bde 2024-11-15T22:13:00,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/ca59894353b449e7bb0fca547cc2a034, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/a58ad782868840fdb32939434273e3bb, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/c2dd20c15c394b4aac6b5b3e5e792574, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/0b03fe7d1c7a4941b914f81b49e82518, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/837b856eb1624800bd4c9b3d6d9bef99, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/9fd49f1133214459817687985eebd900, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/c03829331ee94090a10c030d02275672, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/7ec9644dab454995b88dd77b409649ab, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/5de3b874e7f6413f833390fc0232a724, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/81b500ab8a3243d3b1b6877d7c627f9e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/6c9acbbce1094ff0b44fef572f44d04f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/9dfcc9d4e0ee47c3be00d58f59bfaa3a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/68840900fe6d4a24a6e7447d63951aee, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/6781176e51e7464ba4057bc3726a9736, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/12a97b8cf6514472818a2c0cc718aec6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/d5f7e547901f45379599a692633e4605, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/45f98c1f28b6491b930d9b64dd3ee7a8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/16e13160bc674843b2869c5eef4264ec, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/1d8ff77694a84c07a3039e6aa1501867, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/e0a6fcaa4fa447e387db124707200365, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/7ada511f242748e996d2c4b83360aa66, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/ecd2f7f273f54efea53d6e9d5392322f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/1f2b495fd6344e18b3316c12ce549d34, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/a2394b57baa147fa84e10e3a2f1a6a1c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/98bec04c95b24fc284eee421e672abb1, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/249fe8d760da4ce58d6dc2f72dd15bf2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/11d9ba5a8d29489ebd6d93dc3b59193b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/24ee51b7a6de41e8a133c570f1349c1a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/c58f2994df434eaf8ffc4a6e042888b6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/a7d2df106a87461581afbebb65bcaf2c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/24d4cd96a54946b2818f26eaf11750d6] to archive 2024-11-15T22:13:00,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:13:00,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/ca59894353b449e7bb0fca547cc2a034 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/ca59894353b449e7bb0fca547cc2a034 2024-11-15T22:13:00,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/a58ad782868840fdb32939434273e3bb to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/a58ad782868840fdb32939434273e3bb 2024-11-15T22:13:00,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/c2dd20c15c394b4aac6b5b3e5e792574 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/c2dd20c15c394b4aac6b5b3e5e792574 2024-11-15T22:13:00,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/0b03fe7d1c7a4941b914f81b49e82518 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/0b03fe7d1c7a4941b914f81b49e82518 2024-11-15T22:13:00,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/837b856eb1624800bd4c9b3d6d9bef99 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/837b856eb1624800bd4c9b3d6d9bef99 2024-11-15T22:13:00,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/9fd49f1133214459817687985eebd900 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/9fd49f1133214459817687985eebd900 2024-11-15T22:13:00,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/c03829331ee94090a10c030d02275672 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/c03829331ee94090a10c030d02275672 2024-11-15T22:13:00,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/7ec9644dab454995b88dd77b409649ab to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/7ec9644dab454995b88dd77b409649ab 2024-11-15T22:13:00,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/5de3b874e7f6413f833390fc0232a724 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/5de3b874e7f6413f833390fc0232a724 2024-11-15T22:13:00,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/81b500ab8a3243d3b1b6877d7c627f9e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/81b500ab8a3243d3b1b6877d7c627f9e 2024-11-15T22:13:00,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/6c9acbbce1094ff0b44fef572f44d04f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/6c9acbbce1094ff0b44fef572f44d04f 2024-11-15T22:13:00,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/9dfcc9d4e0ee47c3be00d58f59bfaa3a to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/9dfcc9d4e0ee47c3be00d58f59bfaa3a 2024-11-15T22:13:00,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/68840900fe6d4a24a6e7447d63951aee to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/68840900fe6d4a24a6e7447d63951aee 2024-11-15T22:13:00,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/6781176e51e7464ba4057bc3726a9736 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/6781176e51e7464ba4057bc3726a9736 2024-11-15T22:13:00,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/12a97b8cf6514472818a2c0cc718aec6 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/12a97b8cf6514472818a2c0cc718aec6 2024-11-15T22:13:00,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/d5f7e547901f45379599a692633e4605 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/d5f7e547901f45379599a692633e4605 2024-11-15T22:13:00,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/45f98c1f28b6491b930d9b64dd3ee7a8 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/45f98c1f28b6491b930d9b64dd3ee7a8 2024-11-15T22:13:00,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/16e13160bc674843b2869c5eef4264ec to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/16e13160bc674843b2869c5eef4264ec 2024-11-15T22:13:00,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/1d8ff77694a84c07a3039e6aa1501867 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/1d8ff77694a84c07a3039e6aa1501867 2024-11-15T22:13:00,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/e0a6fcaa4fa447e387db124707200365 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/e0a6fcaa4fa447e387db124707200365 2024-11-15T22:13:00,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/7ada511f242748e996d2c4b83360aa66 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/7ada511f242748e996d2c4b83360aa66 2024-11-15T22:13:00,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/ecd2f7f273f54efea53d6e9d5392322f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/ecd2f7f273f54efea53d6e9d5392322f 2024-11-15T22:13:00,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/1f2b495fd6344e18b3316c12ce549d34 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/1f2b495fd6344e18b3316c12ce549d34 2024-11-15T22:13:00,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/a2394b57baa147fa84e10e3a2f1a6a1c to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/a2394b57baa147fa84e10e3a2f1a6a1c 2024-11-15T22:13:00,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/98bec04c95b24fc284eee421e672abb1 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/98bec04c95b24fc284eee421e672abb1 2024-11-15T22:13:00,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/249fe8d760da4ce58d6dc2f72dd15bf2 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/249fe8d760da4ce58d6dc2f72dd15bf2 2024-11-15T22:13:00,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/11d9ba5a8d29489ebd6d93dc3b59193b to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/11d9ba5a8d29489ebd6d93dc3b59193b 2024-11-15T22:13:00,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/24ee51b7a6de41e8a133c570f1349c1a to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/24ee51b7a6de41e8a133c570f1349c1a 2024-11-15T22:13:00,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/c58f2994df434eaf8ffc4a6e042888b6 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/c58f2994df434eaf8ffc4a6e042888b6 2024-11-15T22:13:00,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/a7d2df106a87461581afbebb65bcaf2c to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/a7d2df106a87461581afbebb65bcaf2c 2024-11-15T22:13:00,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/24d4cd96a54946b2818f26eaf11750d6 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/24d4cd96a54946b2818f26eaf11750d6 2024-11-15T22:13:00,384 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/recovered.edits/465.seqid, newMaxSeqId=465, maxSeqId=1 2024-11-15T22:13:00,385 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. 2024-11-15T22:13:00,385 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 5d997b3c24773a2e95e850dd849536df: Waiting for close lock at 1731708779426Running coprocessor pre-close hooks at 1731708779426Disabling compacts and flushes for region at 1731708779426Disabling writes for close at 1731708779426Obtaining lock to block concurrent updates at 1731708779426Preparing flush snapshotting stores in 5d997b3c24773a2e95e850dd849536df at 1731708779426Finished memstore snapshotting TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731708779426Flushing stores of TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df. at 1731708779427 (+1 ms)Flushing 5d997b3c24773a2e95e850dd849536df/A: creating writer at 1731708779427Flushing 5d997b3c24773a2e95e850dd849536df/A: appending metadata at 1731708779430 (+3 ms)Flushing 5d997b3c24773a2e95e850dd849536df/A: closing flushed file at 1731708779430Flushing 5d997b3c24773a2e95e850dd849536df/B: creating writer at 1731708779839 (+409 ms)Flushing 5d997b3c24773a2e95e850dd849536df/B: appending metadata at 1731708779841 (+2 ms)Flushing 5d997b3c24773a2e95e850dd849536df/B: closing flushed file at 1731708779841Flushing 5d997b3c24773a2e95e850dd849536df/C: creating writer at 1731708779849 (+8 ms)Flushing 5d997b3c24773a2e95e850dd849536df/C: appending metadata at 1731708779851 (+2 ms)Flushing 5d997b3c24773a2e95e850dd849536df/C: closing flushed file at 1731708779851Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@780ab360: reopening flushed file at 1731708780259 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d94b1c9: reopening flushed file at 1731708780264 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1877e917: reopening flushed file at 1731708780268 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 5d997b3c24773a2e95e850dd849536df in 847ms, sequenceid=462, compaction requested=true at 1731708780273 (+5 ms)Writing region close event to WAL at 1731708780380 (+107 ms)Running coprocessor post-close hooks at 1731708780384 (+4 ms)Closed at 1731708780385 (+1 ms) 2024-11-15T22:13:00,387 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 5d997b3c24773a2e95e850dd849536df 2024-11-15T22:13:00,388 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=5d997b3c24773a2e95e850dd849536df, regionState=CLOSED 2024-11-15T22:13:00,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 because future has completed 2024-11-15T22:13:00,392 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-15T22:13:00,392 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 5d997b3c24773a2e95e850dd849536df, server=ae3311ea5ccb,37917,1731708686516 in 1.1170 sec 2024-11-15T22:13:00,394 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-15T22:13:00,394 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5d997b3c24773a2e95e850dd849536df, UNASSIGN in 1.1220 sec 2024-11-15T22:13:00,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-15T22:13:00,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1260 sec 2024-11-15T22:13:00,397 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708780397"}]},"ts":"1731708780397"} 2024-11-15T22:13:00,398 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T22:13:00,398 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T22:13:00,400 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1360 sec 2024-11-15T22:13:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T22:13:00,407 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:00,408 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T22:13:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:00,409 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:00,410 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-15T22:13:00,411 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df 2024-11-15T22:13:00,413 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A, FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B, FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C, FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/recovered.edits] 2024-11-15T22:13:00,416 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/462ef0f73c1e452d99b5581e442479b3 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/462ef0f73c1e452d99b5581e442479b3 2024-11-15T22:13:00,417 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/7cb0b40db3e148ceae54361463790e17 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/7cb0b40db3e148ceae54361463790e17 2024-11-15T22:13:00,418 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/f78b88f5c657488180fb9c2030a439b5 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/A/f78b88f5c657488180fb9c2030a439b5 2024-11-15T22:13:00,420 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4c3d2faa82a043fa90d82f3a41bbd89b to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/4c3d2faa82a043fa90d82f3a41bbd89b 2024-11-15T22:13:00,421 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/550968a7f55b44f8a808bed67a81e2d8 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/550968a7f55b44f8a808bed67a81e2d8 2024-11-15T22:13:00,422 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/dcead682f55c41139df133e627fae895 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/B/dcead682f55c41139df133e627fae895 2024-11-15T22:13:00,424 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/27666c1ffcfc47d29504bd9fb8205035 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/27666c1ffcfc47d29504bd9fb8205035 2024-11-15T22:13:00,425 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/985ea7a76b884eedb7cb2cce57ed2063 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/985ea7a76b884eedb7cb2cce57ed2063 2024-11-15T22:13:00,426 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/9dff5c3b5c944b88837f11f3b3598b6e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/C/9dff5c3b5c944b88837f11f3b3598b6e 2024-11-15T22:13:00,428 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/recovered.edits/465.seqid to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df/recovered.edits/465.seqid 2024-11-15T22:13:00,429 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/5d997b3c24773a2e95e850dd849536df 2024-11-15T22:13:00,429 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T22:13:00,431 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:00,433 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T22:13:00,435 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T22:13:00,436 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:00,436 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T22:13:00,436 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731708780436"}]},"ts":"9223372036854775807"} 2024-11-15T22:13:00,438 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T22:13:00,438 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5d997b3c24773a2e95e850dd849536df, NAME => 'TestAcidGuarantees,,1731708748468.5d997b3c24773a2e95e850dd849536df.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T22:13:00,438 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T22:13:00,438 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731708780438"}]},"ts":"9223372036854775807"} 2024-11-15T22:13:00,440 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T22:13:00,440 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:00,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-11-15T22:13:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-15T22:13:00,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T22:13:00,518 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:00,530 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238 (was 241), OpenFileDescriptor=449 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=632 (was 699), ProcessCount=11 (was 11), AvailableMemoryMB=3440 (was 2747) - AvailableMemoryMB LEAK? - 2024-11-15T22:13:00,540 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=632, ProcessCount=11, AvailableMemoryMB=3440 2024-11-15T22:13:00,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T22:13:00,542 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:13:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:00,544 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T22:13:00,544 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:00,544 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-11-15T22:13:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T22:13:00,545 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T22:13:00,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742160_1336 (size=959) 2024-11-15T22:13:00,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T22:13:00,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T22:13:00,955 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31 2024-11-15T22:13:00,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742161_1337 (size=53) 2024-11-15T22:13:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T22:13:01,362 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:13:01,362 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2a98cc30238e089ac8df02a2f0745572, disabling compactions & flushes 2024-11-15T22:13:01,362 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:01,362 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:01,362 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. after waiting 0 ms 2024-11-15T22:13:01,362 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:01,362 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:01,362 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2a98cc30238e089ac8df02a2f0745572: Waiting for close lock at 1731708781362Disabling compacts and flushes for region at 1731708781362Disabling writes for close at 1731708781362Writing region close event to WAL at 1731708781362Closed at 1731708781362 2024-11-15T22:13:01,363 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T22:13:01,364 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731708781363"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708781363"}]},"ts":"1731708781363"} 2024-11-15T22:13:01,365 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T22:13:01,366 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T22:13:01,366 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708781366"}]},"ts":"1731708781366"} 2024-11-15T22:13:01,368 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T22:13:01,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a98cc30238e089ac8df02a2f0745572, ASSIGN}] 2024-11-15T22:13:01,369 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a98cc30238e089ac8df02a2f0745572, ASSIGN 2024-11-15T22:13:01,370 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a98cc30238e089ac8df02a2f0745572, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,37917,1731708686516; forceNewPlan=false, retain=false 2024-11-15T22:13:01,521 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=2a98cc30238e089ac8df02a2f0745572, regionState=OPENING, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:01,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a98cc30238e089ac8df02a2f0745572, ASSIGN because future has completed 2024-11-15T22:13:01,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516}] 2024-11-15T22:13:01,678 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:01,678 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:13:01,678 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:01,679 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:13:01,679 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:01,679 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:01,680 INFO [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:01,681 INFO [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a98cc30238e089ac8df02a2f0745572 columnFamilyName A 2024-11-15T22:13:01,681 DEBUG [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:01,682 INFO [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] regionserver.HStore(327): Store=2a98cc30238e089ac8df02a2f0745572/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:13:01,682 INFO [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:01,683 INFO [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a98cc30238e089ac8df02a2f0745572 columnFamilyName B 2024-11-15T22:13:01,683 DEBUG [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:01,683 INFO [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] regionserver.HStore(327): Store=2a98cc30238e089ac8df02a2f0745572/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:13:01,683 INFO [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:01,684 INFO [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a98cc30238e089ac8df02a2f0745572 columnFamilyName C 2024-11-15T22:13:01,684 DEBUG [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:01,684 INFO [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] regionserver.HStore(327): Store=2a98cc30238e089ac8df02a2f0745572/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:13:01,685 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:01,685 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:01,685 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T22:13:01,686 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:01,686 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:01,687 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:13:01,688 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:01,690 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:13:01,690 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened 2a98cc30238e089ac8df02a2f0745572; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64361303, jitterRate=-0.04094184935092926}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:13:01,690 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:01,691 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for 2a98cc30238e089ac8df02a2f0745572: Running coprocessor pre-open hook at 1731708781679Writing region info on filesystem at 1731708781679Initializing all the Stores at 1731708781679Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708781680 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708781680Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708781680Cleaning up temporary data from old regions at 1731708781686 (+6 ms)Running coprocessor post-open hooks at 1731708781690 (+4 ms)Region opened successfully at 1731708781691 (+1 ms) 2024-11-15T22:13:01,692 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., pid=93, masterSystemTime=1731708781675 2024-11-15T22:13:01,694 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:01,694 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:01,695 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=2a98cc30238e089ac8df02a2f0745572, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:01,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 because future has completed 2024-11-15T22:13:01,699 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-15T22:13:01,699 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 in 174 msec 2024-11-15T22:13:01,700 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-15T22:13:01,700 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a98cc30238e089ac8df02a2f0745572, ASSIGN in 331 msec 2024-11-15T22:13:01,701 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T22:13:01,701 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708781701"}]},"ts":"1731708781701"} 2024-11-15T22:13:01,703 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T22:13:01,704 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T22:13:01,705 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1620 sec 2024-11-15T22:13:02,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-15T22:13:02,696 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:02,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@284db418, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:02,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:02,698 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:02,699 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:02,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:02,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:02,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b252bb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:02,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:02,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:02,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:02,701 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:02,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c004e9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:02,702 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:02,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:02,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:02,704 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:02,705 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T22:13:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is ae3311ea5ccb,37303,1731708685620 2024-11-15T22:13:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5782b977 2024-11-15T22:13:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T22:13:02,707 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T22:13:02,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T22:13:02,709 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:13:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:02,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742162_1338 (size=995) 2024-11-15T22:13:03,124 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-15T22:13:03,124 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-15T22:13:03,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T22:13:03,128 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a98cc30238e089ac8df02a2f0745572, REOPEN/MOVE}] 2024-11-15T22:13:03,130 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a98cc30238e089ac8df02a2f0745572, REOPEN/MOVE 2024-11-15T22:13:03,131 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=2a98cc30238e089ac8df02a2f0745572, regionState=CLOSING, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:03,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a98cc30238e089ac8df02a2f0745572, REOPEN/MOVE because future has completed 2024-11-15T22:13:03,138 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T22:13:03,138 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516}] 2024-11-15T22:13:03,291 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:03,291 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T22:13:03,291 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 2a98cc30238e089ac8df02a2f0745572, disabling compactions & flushes 2024-11-15T22:13:03,291 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:03,291 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:03,291 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. after waiting 0 ms 2024-11-15T22:13:03,291 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:03,295 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:13:03,295 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:03,296 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 2a98cc30238e089ac8df02a2f0745572: Waiting for close lock at 1731708783291Running coprocessor pre-close hooks at 1731708783291Disabling compacts and flushes for region at 1731708783291Disabling writes for close at 1731708783291Writing region close event to WAL at 1731708783291Running coprocessor post-close hooks at 1731708783295 (+4 ms)Closed at 1731708783295 2024-11-15T22:13:03,296 WARN [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: 2a98cc30238e089ac8df02a2f0745572 to self. 2024-11-15T22:13:03,298 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:03,298 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=2a98cc30238e089ac8df02a2f0745572, regionState=CLOSED 2024-11-15T22:13:03,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 because future has completed 2024-11-15T22:13:03,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-15T22:13:03,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 in 162 msec 2024-11-15T22:13:03,303 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a98cc30238e089ac8df02a2f0745572, REOPEN/MOVE; state=CLOSED, location=ae3311ea5ccb,37917,1731708686516; forceNewPlan=false, retain=true 2024-11-15T22:13:03,454 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=2a98cc30238e089ac8df02a2f0745572, regionState=OPENING, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:03,456 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a98cc30238e089ac8df02a2f0745572, REOPEN/MOVE because future has completed 2024-11-15T22:13:03,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516}] 2024-11-15T22:13:03,611 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:03,612 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:13:03,612 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:03,612 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:13:03,612 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:03,612 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:03,614 INFO [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:03,614 INFO [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a98cc30238e089ac8df02a2f0745572 columnFamilyName A 2024-11-15T22:13:03,615 DEBUG [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:03,616 INFO [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] regionserver.HStore(327): Store=2a98cc30238e089ac8df02a2f0745572/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:13:03,616 INFO [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:03,617 INFO [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a98cc30238e089ac8df02a2f0745572 columnFamilyName B 2024-11-15T22:13:03,617 DEBUG [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:03,617 INFO [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] regionserver.HStore(327): Store=2a98cc30238e089ac8df02a2f0745572/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:13:03,617 INFO [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:03,618 INFO [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a98cc30238e089ac8df02a2f0745572 columnFamilyName C 2024-11-15T22:13:03,618 DEBUG [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:03,618 INFO [StoreOpener-2a98cc30238e089ac8df02a2f0745572-1 {}] regionserver.HStore(327): Store=2a98cc30238e089ac8df02a2f0745572/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:13:03,618 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:03,619 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:03,620 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:03,620 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:03,621 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:03,621 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:13:03,622 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:03,623 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened 2a98cc30238e089ac8df02a2f0745572; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74082259, jitterRate=0.10391168296337128}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:13:03,623 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:03,623 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for 2a98cc30238e089ac8df02a2f0745572: Running coprocessor pre-open hook at 1731708783613Writing region info on filesystem at 1731708783613Initializing all the Stores at 1731708783613Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708783613Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708783613Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708783613Cleaning up temporary data from old regions at 1731708783621 (+8 ms)Running coprocessor post-open hooks at 1731708783623 (+2 ms)Region opened successfully at 1731708783623 2024-11-15T22:13:03,624 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., pid=98, masterSystemTime=1731708783608 2024-11-15T22:13:03,626 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:03,626 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:03,627 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=2a98cc30238e089ac8df02a2f0745572, regionState=OPEN, openSeqNum=5, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:03,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 because future has completed 2024-11-15T22:13:03,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-11-15T22:13:03,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 in 173 msec 2024-11-15T22:13:03,632 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-15T22:13:03,632 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a98cc30238e089ac8df02a2f0745572, REOPEN/MOVE in 502 msec 2024-11-15T22:13:03,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-15T22:13:03,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 507 msec 2024-11-15T22:13:03,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 925 msec 2024-11-15T22:13:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-15T22:13:03,638 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:03,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79018c7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,639 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:03,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:03,640 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d6a3c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:03,641 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:03,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@239f9edd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@725f7689, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,642 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:03,643 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:03,643 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:03,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:03,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:03,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f0a4ec6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:03,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:03,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:03,645 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3098dbc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@427fac46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,646 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:03,646 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:03,647 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:03,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:03,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:03,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@467ece2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:03,648 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:03,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ddfb0c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a741c6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,650 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:03,650 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:03,651 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@585f4bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:03,652 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c2688cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58c0c28c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,653 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:03,653 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:03,654 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:03,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:03,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:03,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c248ad5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:03,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:03,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:03,655 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:03,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2610e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2108eda1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,657 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:03,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:03,657 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:03,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:03,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:03,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f74a7f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:03,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:03,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:03,658 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:03,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@509b0194, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@354663d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:03,660 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:03,660 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:03,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4840d8dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:03,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:03,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:03,661 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:03,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74fd71c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bbfc9f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,663 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:03,663 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:03,663 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:03,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:03,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:03,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b908d52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:03,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:03,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:03,665 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:03,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50e4033c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c87703e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:03,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:03,668 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26da8715, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:03,669 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19097b42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7919b73e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:03,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:03,671 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:03,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:03,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:03,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19a15a02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:03,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:03,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:03,672 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77eda849, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:03,674 DEBUG [Thread-1548 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:03,674 DEBUG [Thread-1555 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:03,674 DEBUG [Thread-1544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:03,674 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:13:03,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:03,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:03,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:03,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:03,676 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:03,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-15T22:13:03,678 DEBUG [Thread-1552 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T22:13:03,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:13:03,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:13:03,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:03,681 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:13:03,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:13:03,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:03,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:03,683 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:13:03,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:13:03,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:03,685 DEBUG [Thread-1557 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:03,685 DEBUG [Thread-1563 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:03,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:13:03,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:03,687 DEBUG [Thread-1563 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:03,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:03,687 DEBUG [Thread-1563 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:03,687 DEBUG [Thread-1550 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:03,688 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:03,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:03,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:03,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:13:03,690 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:03,690 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:03,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:13:03,694 DEBUG [Thread-1546 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:13:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:03,695 DEBUG [Thread-1561 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:03,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:03,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:03,696 DEBUG [Thread-1559 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:03,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:03,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:03,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:03,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:03,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:03,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:13:03,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:13:03,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:13:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:03,712 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:03,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T22:13:03,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115241b7f1a53f14b4d9f765ac789484d62_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708783688/Put/seqid=0 2024-11-15T22:13:03,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742163_1339 (size=14594) 2024-11-15T22:13:03,777 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:03,781 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115241b7f1a53f14b4d9f765ac789484d62_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115241b7f1a53f14b4d9f765ac789484d62_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:03,785 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/8784d22482454ff3a4c0bee3a0c864e8, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:03,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/8784d22482454ff3a4c0bee3a0c864e8 is 175, key is test_row_0/A:col10/1731708783688/Put/seqid=0 2024-11-15T22:13:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T22:13:03,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708843775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:03,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708843776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:03,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:03,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742164_1340 (size=39549) 2024-11-15T22:13:03,792 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/8784d22482454ff3a4c0bee3a0c864e8 2024-11-15T22:13:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:03,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:03,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:03,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:03,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708843785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:03,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708843786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:03,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708843786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:03,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:03,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/ecbef8b4599449b29bc1f8512d0e3252 is 50, key is test_row_0/B:col10/1731708783688/Put/seqid=0 2024-11-15T22:13:03,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742165_1341 (size=12001) 2024-11-15T22:13:03,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/ecbef8b4599449b29bc1f8512d0e3252 2024-11-15T22:13:03,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T22:13:03,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:03,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:03,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:03,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:03,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:03,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/e1de6822faa24552a7a9385c709c6584 is 50, key is test_row_0/C:col10/1731708783688/Put/seqid=0 2024-11-15T22:13:03,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742166_1342 (size=12001) 2024-11-15T22:13:03,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/e1de6822faa24552a7a9385c709c6584 2024-11-15T22:13:03,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/8784d22482454ff3a4c0bee3a0c864e8 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/8784d22482454ff3a4c0bee3a0c864e8 2024-11-15T22:13:03,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/8784d22482454ff3a4c0bee3a0c864e8, entries=200, sequenceid=17, filesize=38.6 K 2024-11-15T22:13:03,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/ecbef8b4599449b29bc1f8512d0e3252 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/ecbef8b4599449b29bc1f8512d0e3252 2024-11-15T22:13:03,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:03,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708843896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:03,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:03,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:03,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:03,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/ecbef8b4599449b29bc1f8512d0e3252, entries=150, sequenceid=17, filesize=11.7 K 2024-11-15T22:13:03,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/e1de6822faa24552a7a9385c709c6584 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e1de6822faa24552a7a9385c709c6584 2024-11-15T22:13:03,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e1de6822faa24552a7a9385c709c6584, entries=150, sequenceid=17, filesize=11.7 K 2024-11-15T22:13:03,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 2a98cc30238e089ac8df02a2f0745572 in 203ms, sequenceid=17, compaction requested=false 2024-11-15T22:13:03,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:03,918 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:03,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T22:13:03,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:03,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708843942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:03,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:03,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708843943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:03,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c5ec27426c4a4141bcaa6042c67710aa_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708783751/Put/seqid=0 2024-11-15T22:13:03,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:03,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708843945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:03,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:03,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:03,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:03,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:03,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708843949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:03,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:03,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T22:13:03,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:03,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:03,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:03,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:03,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:03,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:03,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T22:13:04,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742167_1343 (size=17034) 2024-11-15T22:13:04,005 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:04,009 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c5ec27426c4a4141bcaa6042c67710aa_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c5ec27426c4a4141bcaa6042c67710aa_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:04,010 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/ca656a2279f14de6a1e9a56c2ce96944, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:04,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/ca656a2279f14de6a1e9a56c2ce96944 is 175, key is test_row_0/A:col10/1731708783751/Put/seqid=0 2024-11-15T22:13:04,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742168_1344 (size=48139) 2024-11-15T22:13:04,031 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=43, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/ca656a2279f14de6a1e9a56c2ce96944 2024-11-15T22:13:04,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/11d920103fde40dda4f245e8282deddc is 50, key is test_row_0/B:col10/1731708783751/Put/seqid=0 2024-11-15T22:13:04,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742169_1345 (size=12001) 2024-11-15T22:13:04,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/11d920103fde40dda4f245e8282deddc 2024-11-15T22:13:04,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708844066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:04,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708844067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:04,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708844067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:04,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708844073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:04,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/6cb1c455821c4a2489802d9fe9f79565 is 50, key is test_row_0/C:col10/1731708783751/Put/seqid=0 2024-11-15T22:13:04,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742170_1346 (size=12001) 2024-11-15T22:13:04,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:04,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708844117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:04,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/6cb1c455821c4a2489802d9fe9f79565 2024-11-15T22:13:04,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/ca656a2279f14de6a1e9a56c2ce96944 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/ca656a2279f14de6a1e9a56c2ce96944 2024-11-15T22:13:04,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/ca656a2279f14de6a1e9a56c2ce96944, entries=250, sequenceid=43, filesize=47.0 K 2024-11-15T22:13:04,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/11d920103fde40dda4f245e8282deddc as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/11d920103fde40dda4f245e8282deddc 2024-11-15T22:13:04,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T22:13:04,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:04,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:04,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:04,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:04,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:04,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:04,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/11d920103fde40dda4f245e8282deddc, entries=150, sequenceid=43, filesize=11.7 K 2024-11-15T22:13:04,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/6cb1c455821c4a2489802d9fe9f79565 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/6cb1c455821c4a2489802d9fe9f79565 2024-11-15T22:13:04,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/6cb1c455821c4a2489802d9fe9f79565, entries=150, sequenceid=43, filesize=11.7 K 2024-11-15T22:13:04,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 2a98cc30238e089ac8df02a2f0745572 in 245ms, sequenceid=43, compaction requested=false 2024-11-15T22:13:04,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:04,294 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:04,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T22:13:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:04,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T22:13:04,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T22:13:04,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115de61289ef81946c8922fb0b872701907_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708783938/Put/seqid=0 2024-11-15T22:13:04,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:04,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:04,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:04,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:04,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:04,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742171_1347 (size=14594) 2024-11-15T22:13:04,329 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:04,337 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115de61289ef81946c8922fb0b872701907_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115de61289ef81946c8922fb0b872701907_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:04,338 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/ddee843a9f4f4b62ac15939df7b51450, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:04,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/ddee843a9f4f4b62ac15939df7b51450 is 175, key is test_row_0/A:col10/1731708783938/Put/seqid=0 2024-11-15T22:13:04,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742172_1348 (size=39549) 2024-11-15T22:13:04,374 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/ddee843a9f4f4b62ac15939df7b51450 2024-11-15T22:13:04,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/839d7cb9bf6c49b0a8fd75257a3b619c is 50, key is test_row_0/B:col10/1731708783938/Put/seqid=0 2024-11-15T22:13:04,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:04,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708844373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:04,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:04,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:04,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708844379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:04,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708844381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:04,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:04,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:04,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708844383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:04,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:04,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742173_1349 (size=12001) 2024-11-15T22:13:04,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:04,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708844437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:04,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:04,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T22:13:04,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:04,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:04,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:04,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:04,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:04,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708844496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:04,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:04,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:04,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708844506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:04,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708844506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:04,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708844506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:04,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T22:13:04,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:04,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:04,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:04,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:04,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:04,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:04,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708844717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:04,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:04,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:04,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708844726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:04,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708844726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:04,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:04,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:04,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708844727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:04,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:04,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:04,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T22:13:04,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:04,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:04,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:04,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:04,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:04,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:04,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/839d7cb9bf6c49b0a8fd75257a3b619c 2024-11-15T22:13:04,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T22:13:04,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/a59b243fc6064d0e8974db73d7ab7019 is 50, key is test_row_0/C:col10/1731708783938/Put/seqid=0 2024-11-15T22:13:04,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742174_1350 (size=12001) 2024-11-15T22:13:04,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/a59b243fc6064d0e8974db73d7ab7019 2024-11-15T22:13:04,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/ddee843a9f4f4b62ac15939df7b51450 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/ddee843a9f4f4b62ac15939df7b51450 2024-11-15T22:13:04,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/ddee843a9f4f4b62ac15939df7b51450, entries=200, sequenceid=55, filesize=38.6 K 2024-11-15T22:13:04,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/839d7cb9bf6c49b0a8fd75257a3b619c as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/839d7cb9bf6c49b0a8fd75257a3b619c 2024-11-15T22:13:04,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/839d7cb9bf6c49b0a8fd75257a3b619c, entries=150, sequenceid=55, filesize=11.7 K 2024-11-15T22:13:04,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/a59b243fc6064d0e8974db73d7ab7019 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/a59b243fc6064d0e8974db73d7ab7019 2024-11-15T22:13:04,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/a59b243fc6064d0e8974db73d7ab7019, entries=150, sequenceid=55, filesize=11.7 K 2024-11-15T22:13:04,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2a98cc30238e089ac8df02a2f0745572 in 596ms, sequenceid=55, compaction requested=true 2024-11-15T22:13:04,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:04,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:13:04,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:04,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:13:04,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:13:04,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:13:04,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T22:13:04,892 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:04,893 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:04,893 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127237 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:04,893 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/A is initiating minor compaction (all files) 2024-11-15T22:13:04,893 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/A in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:04,894 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/8784d22482454ff3a4c0bee3a0c864e8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/ca656a2279f14de6a1e9a56c2ce96944, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/ddee843a9f4f4b62ac15939df7b51450] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=124.3 K 2024-11-15T22:13:04,894 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:04,894 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/8784d22482454ff3a4c0bee3a0c864e8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/ca656a2279f14de6a1e9a56c2ce96944, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/ddee843a9f4f4b62ac15939df7b51450] 2024-11-15T22:13:04,894 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 8784d22482454ff3a4c0bee3a0c864e8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731708783688 2024-11-15T22:13:04,895 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting ca656a2279f14de6a1e9a56c2ce96944, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731708783751 2024-11-15T22:13:04,895 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting ddee843a9f4f4b62ac15939df7b51450, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731708783938 2024-11-15T22:13:04,901 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:04,901 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/C is initiating minor compaction (all files) 2024-11-15T22:13:04,901 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/C in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:04,901 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e1de6822faa24552a7a9385c709c6584, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/6cb1c455821c4a2489802d9fe9f79565, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/a59b243fc6064d0e8974db73d7ab7019] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=35.2 K 2024-11-15T22:13:04,902 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1de6822faa24552a7a9385c709c6584, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731708783688 2024-11-15T22:13:04,902 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cb1c455821c4a2489802d9fe9f79565, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731708783751 2024-11-15T22:13:04,903 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:04,903 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting a59b243fc6064d0e8974db73d7ab7019, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731708783938 2024-11-15T22:13:04,908 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115acc82d892ef0489b82737878a238030a_2a98cc30238e089ac8df02a2f0745572 store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:04,910 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115acc82d892ef0489b82737878a238030a_2a98cc30238e089ac8df02a2f0745572, store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:04,910 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115acc82d892ef0489b82737878a238030a_2a98cc30238e089ac8df02a2f0745572 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:04,914 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#C#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:04,914 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/1ac38a6212e74467ba6696c8127bc211 is 50, key is test_row_0/C:col10/1731708783938/Put/seqid=0 2024-11-15T22:13:04,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T22:13:04,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:04,942 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T22:13:04,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742175_1351 (size=4469) 2024-11-15T22:13:04,954 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#A#compaction#293 average throughput is 0.48 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:04,954 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/e6f1f0478abd43408ee23fb350b94efd is 175, key is test_row_0/A:col10/1731708783938/Put/seqid=0 2024-11-15T22:13:04,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:04,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:04,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742176_1352 (size=12104) 2024-11-15T22:13:04,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cfc02a9f9a8a4e688dc5f94619c58218_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708784380/Put/seqid=0 2024-11-15T22:13:04,991 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/1ac38a6212e74467ba6696c8127bc211 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/1ac38a6212e74467ba6696c8127bc211 2024-11-15T22:13:05,004 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/C of 2a98cc30238e089ac8df02a2f0745572 into 1ac38a6212e74467ba6696c8127bc211(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:05,004 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:05,004 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/C, priority=13, startTime=1731708784890; duration=0sec 2024-11-15T22:13:05,004 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:05,004 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:C 2024-11-15T22:13:05,005 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:05,005 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:05,006 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/B is initiating minor compaction (all files) 2024-11-15T22:13:05,006 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/B in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:05,006 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/ecbef8b4599449b29bc1f8512d0e3252, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/11d920103fde40dda4f245e8282deddc, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/839d7cb9bf6c49b0a8fd75257a3b619c] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=35.2 K 2024-11-15T22:13:05,010 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecbef8b4599449b29bc1f8512d0e3252, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731708783688 2024-11-15T22:13:05,011 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11d920103fde40dda4f245e8282deddc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731708783751 2024-11-15T22:13:05,011 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 839d7cb9bf6c49b0a8fd75257a3b619c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731708783938 2024-11-15T22:13:05,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742177_1353 (size=31058) 2024-11-15T22:13:05,051 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#B#compaction#296 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:05,052 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/fd225ccb4a724ee1bc317501584932ff is 50, key is test_row_0/B:col10/1731708783938/Put/seqid=0 2024-11-15T22:13:05,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742178_1354 (size=12154) 2024-11-15T22:13:05,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:05,073 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cfc02a9f9a8a4e688dc5f94619c58218_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cfc02a9f9a8a4e688dc5f94619c58218_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:05,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/f97e3e894ec54212b112e725710f8f3a, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:05,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/f97e3e894ec54212b112e725710f8f3a is 175, key is test_row_0/A:col10/1731708784380/Put/seqid=0 2024-11-15T22:13:05,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708845048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708845056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742179_1355 (size=12104) 2024-11-15T22:13:05,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742180_1356 (size=30955) 2024-11-15T22:13:05,108 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/f97e3e894ec54212b112e725710f8f3a 2024-11-15T22:13:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708845071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708845082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708845083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,119 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/fd225ccb4a724ee1bc317501584932ff as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/fd225ccb4a724ee1bc317501584932ff 2024-11-15T22:13:05,127 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/B of 2a98cc30238e089ac8df02a2f0745572 into fd225ccb4a724ee1bc317501584932ff(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:05,127 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:05,127 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/B, priority=13, startTime=1731708784890; duration=0sec 2024-11-15T22:13:05,128 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:05,128 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:B 2024-11-15T22:13:05,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/ba9d5a6ec2aa450b8b69c42b0d77702c is 50, key is test_row_0/B:col10/1731708784380/Put/seqid=0 2024-11-15T22:13:05,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742181_1357 (size=12001) 2024-11-15T22:13:05,185 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/ba9d5a6ec2aa450b8b69c42b0d77702c 2024-11-15T22:13:05,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/0e564e128ed04264819162fdc68823c0 is 50, key is test_row_0/C:col10/1731708784380/Put/seqid=0 2024-11-15T22:13:05,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708845190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708845190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742182_1358 (size=12001) 2024-11-15T22:13:05,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708845216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708845227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708845227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708845417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708845417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,450 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/e6f1f0478abd43408ee23fb350b94efd as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/e6f1f0478abd43408ee23fb350b94efd 2024-11-15T22:13:05,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708845438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708845438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708845438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,466 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/A of 2a98cc30238e089ac8df02a2f0745572 into e6f1f0478abd43408ee23fb350b94efd(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:05,466 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:05,466 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/A, priority=13, startTime=1731708784890; duration=0sec 2024-11-15T22:13:05,466 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:05,466 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:A 2024-11-15T22:13:05,610 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/0e564e128ed04264819162fdc68823c0 2024-11-15T22:13:05,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/f97e3e894ec54212b112e725710f8f3a as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/f97e3e894ec54212b112e725710f8f3a 2024-11-15T22:13:05,621 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/f97e3e894ec54212b112e725710f8f3a, entries=150, sequenceid=79, filesize=30.2 K 2024-11-15T22:13:05,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/ba9d5a6ec2aa450b8b69c42b0d77702c as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/ba9d5a6ec2aa450b8b69c42b0d77702c 2024-11-15T22:13:05,633 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/ba9d5a6ec2aa450b8b69c42b0d77702c, entries=150, sequenceid=79, filesize=11.7 K 2024-11-15T22:13:05,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/0e564e128ed04264819162fdc68823c0 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/0e564e128ed04264819162fdc68823c0 2024-11-15T22:13:05,643 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/0e564e128ed04264819162fdc68823c0, entries=150, sequenceid=79, filesize=11.7 K 2024-11-15T22:13:05,644 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 2a98cc30238e089ac8df02a2f0745572 in 702ms, sequenceid=79, compaction requested=false 2024-11-15T22:13:05,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:05,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:05,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-15T22:13:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-15T22:13:05,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-15T22:13:05,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9670 sec 2024-11-15T22:13:05,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.9780 sec 2024-11-15T22:13:05,742 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:05,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T22:13:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:05,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111527a4c9569abd4e32a12c1258f3bd6660_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708785020/Put/seqid=0 2024-11-15T22:13:05,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742183_1359 (size=12154) 2024-11-15T22:13:05,804 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:05,818 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111527a4c9569abd4e32a12c1258f3bd6660_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111527a4c9569abd4e32a12c1258f3bd6660_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:05,823 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/97034a49ee044ebf9918835cbded17b0, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:05,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/97034a49ee044ebf9918835cbded17b0 is 175, key is test_row_0/A:col10/1731708785020/Put/seqid=0 2024-11-15T22:13:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T22:13:05,828 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:05,833 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:13:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-15T22:13:05,835 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:13:05,836 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:13:05,836 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:13:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T22:13:05,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708845807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742184_1360 (size=30955) 2024-11-15T22:13:05,854 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/97034a49ee044ebf9918835cbded17b0 2024-11-15T22:13:05,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708845828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708845841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708845846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708845846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/0602d26d71d2423c9f17015e1c90af94 is 50, key is test_row_0/B:col10/1731708785020/Put/seqid=0 2024-11-15T22:13:05,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742185_1361 (size=12001) 2024-11-15T22:13:05,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/0602d26d71d2423c9f17015e1c90af94 2024-11-15T22:13:05,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/e8a2d4ebe990444989851c71fc37d035 is 50, key is test_row_0/C:col10/1731708785020/Put/seqid=0 2024-11-15T22:13:05,938 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T22:13:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T22:13:05,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708845947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742186_1362 (size=12001) 2024-11-15T22:13:05,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/e8a2d4ebe990444989851c71fc37d035 2024-11-15T22:13:05,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/97034a49ee044ebf9918835cbded17b0 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/97034a49ee044ebf9918835cbded17b0 2024-11-15T22:13:05,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708845967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708845967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708845976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708845977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:05,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T22:13:05,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:05,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:05,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:05,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:05,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:05,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/97034a49ee044ebf9918835cbded17b0, entries=150, sequenceid=95, filesize=30.2 K 2024-11-15T22:13:05,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/0602d26d71d2423c9f17015e1c90af94 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/0602d26d71d2423c9f17015e1c90af94 2024-11-15T22:13:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:06,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/0602d26d71d2423c9f17015e1c90af94, entries=150, sequenceid=95, filesize=11.7 K 2024-11-15T22:13:06,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/e8a2d4ebe990444989851c71fc37d035 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e8a2d4ebe990444989851c71fc37d035 2024-11-15T22:13:06,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e8a2d4ebe990444989851c71fc37d035, entries=150, sequenceid=95, filesize=11.7 K 2024-11-15T22:13:06,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 2a98cc30238e089ac8df02a2f0745572 in 271ms, sequenceid=95, compaction requested=true 2024-11-15T22:13:06,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:06,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:13:06,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:06,013 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:06,013 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:06,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:13:06,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:06,016 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:06,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:13:06,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:06,016 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/B is initiating minor compaction (all files) 2024-11-15T22:13:06,016 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/B in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:06,016 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/fd225ccb4a724ee1bc317501584932ff, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/ba9d5a6ec2aa450b8b69c42b0d77702c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/0602d26d71d2423c9f17015e1c90af94] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=35.3 K 2024-11-15T22:13:06,017 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:06,017 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/A is initiating minor compaction (all files) 2024-11-15T22:13:06,017 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/A in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:06,017 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/e6f1f0478abd43408ee23fb350b94efd, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/f97e3e894ec54212b112e725710f8f3a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/97034a49ee044ebf9918835cbded17b0] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=90.8 K 2024-11-15T22:13:06,017 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:06,017 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/e6f1f0478abd43408ee23fb350b94efd, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/f97e3e894ec54212b112e725710f8f3a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/97034a49ee044ebf9918835cbded17b0] 2024-11-15T22:13:06,018 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting fd225ccb4a724ee1bc317501584932ff, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731708783938 2024-11-15T22:13:06,018 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6f1f0478abd43408ee23fb350b94efd, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731708783938 2024-11-15T22:13:06,019 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting ba9d5a6ec2aa450b8b69c42b0d77702c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731708784362 2024-11-15T22:13:06,019 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting f97e3e894ec54212b112e725710f8f3a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731708784362 2024-11-15T22:13:06,020 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 0602d26d71d2423c9f17015e1c90af94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731708785020 2024-11-15T22:13:06,020 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97034a49ee044ebf9918835cbded17b0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731708785020 2024-11-15T22:13:06,032 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:06,046 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115a4e94a369d2c4bb49bf4105bb788f7aa_2a98cc30238e089ac8df02a2f0745572 store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:06,048 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115a4e94a369d2c4bb49bf4105bb788f7aa_2a98cc30238e089ac8df02a2f0745572, store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:06,048 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a4e94a369d2c4bb49bf4105bb788f7aa_2a98cc30238e089ac8df02a2f0745572 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:06,057 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#B#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:06,058 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/b95fb6097065424ba808150b6be0d294 is 50, key is test_row_0/B:col10/1731708785020/Put/seqid=0 2024-11-15T22:13:06,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742187_1363 (size=4469) 2024-11-15T22:13:06,082 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#A#compaction#302 average throughput is 0.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:06,083 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/daadc4a0f270438e939f3c9b0eec71a7 is 175, key is test_row_0/A:col10/1731708785020/Put/seqid=0 2024-11-15T22:13:06,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742188_1364 (size=12207) 2024-11-15T22:13:06,094 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/b95fb6097065424ba808150b6be0d294 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/b95fb6097065424ba808150b6be0d294 2024-11-15T22:13:06,103 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/B of 2a98cc30238e089ac8df02a2f0745572 into b95fb6097065424ba808150b6be0d294(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:06,103 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:06,103 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/B, priority=13, startTime=1731708786013; duration=0sec 2024-11-15T22:13:06,103 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:06,103 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:B 2024-11-15T22:13:06,103 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:06,104 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:06,104 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/C is initiating minor compaction (all files) 2024-11-15T22:13:06,104 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/C in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:06,105 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/1ac38a6212e74467ba6696c8127bc211, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/0e564e128ed04264819162fdc68823c0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e8a2d4ebe990444989851c71fc37d035] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=35.3 K 2024-11-15T22:13:06,105 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ac38a6212e74467ba6696c8127bc211, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731708783938 2024-11-15T22:13:06,105 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e564e128ed04264819162fdc68823c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731708784362 2024-11-15T22:13:06,106 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting e8a2d4ebe990444989851c71fc37d035, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731708785020 2024-11-15T22:13:06,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742189_1365 (size=31161) 2024-11-15T22:13:06,120 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/daadc4a0f270438e939f3c9b0eec71a7 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/daadc4a0f270438e939f3c9b0eec71a7 2024-11-15T22:13:06,123 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#C#compaction#304 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:06,123 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/6121a0b8c97443e4a10bd7a2397cf401 is 50, key is test_row_0/C:col10/1731708785020/Put/seqid=0 2024-11-15T22:13:06,131 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/A of 2a98cc30238e089ac8df02a2f0745572 into daadc4a0f270438e939f3c9b0eec71a7(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:06,131 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:06,131 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/A, priority=13, startTime=1731708786013; duration=0sec 2024-11-15T22:13:06,131 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:06,131 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:A 2024-11-15T22:13:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T22:13:06,163 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:06,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T22:13:06,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T22:13:06,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:06,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:06,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:06,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:06,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:06,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742190_1366 (size=12207) 2024-11-15T22:13:06,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115adbd8c75031b4b3aa3f00dde1b4d65d0_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708785824/Put/seqid=0 2024-11-15T22:13:06,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708846207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:06,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:06,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708846207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:06,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:06,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:06,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708846208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:06,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:06,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708846210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:06,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:06,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708846221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:06,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742191_1367 (size=14594) 2024-11-15T22:13:06,238 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:06,242 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115adbd8c75031b4b3aa3f00dde1b4d65d0_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115adbd8c75031b4b3aa3f00dde1b4d65d0_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:06,243 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/c449399b7db04829b11f4387138cce07, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:06,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/c449399b7db04829b11f4387138cce07 is 175, key is test_row_0/A:col10/1731708785824/Put/seqid=0 2024-11-15T22:13:06,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742192_1368 (size=39549) 2024-11-15T22:13:06,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T22:13:06,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:06,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:06,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:06,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:06,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:06,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708846326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:06,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:06,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708846337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:06,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708846337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708846337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:06,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708846337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T22:13:06,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T22:13:06,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:06,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:06,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:06,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:06,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:06,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708846537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:06,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708846559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:06,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:06,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708846559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:06,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:06,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:06,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:06,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708846559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:06,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708846559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:06,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:06,589 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/6121a0b8c97443e4a10bd7a2397cf401 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/6121a0b8c97443e4a10bd7a2397cf401 2024-11-15T22:13:06,601 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/C of 2a98cc30238e089ac8df02a2f0745572 into 6121a0b8c97443e4a10bd7a2397cf401(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:06,601 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:06,601 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/C, priority=13, startTime=1731708786016; duration=0sec 2024-11-15T22:13:06,601 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:06,601 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:C 2024-11-15T22:13:06,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T22:13:06,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:06,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:06,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:06,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:06,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:06,684 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/c449399b7db04829b11f4387138cce07 2024-11-15T22:13:06,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/f726a790cf1b438b9b9df990bca620ab is 50, key is test_row_0/B:col10/1731708785824/Put/seqid=0 2024-11-15T22:13:06,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742193_1369 (size=12001) 2024-11-15T22:13:06,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/f726a790cf1b438b9b9df990bca620ab 2024-11-15T22:13:06,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/9a6d6da6ea154adf90575eaab5d22290 is 50, key is test_row_0/C:col10/1731708785824/Put/seqid=0 2024-11-15T22:13:06,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742194_1370 (size=12001) 2024-11-15T22:13:06,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/9a6d6da6ea154adf90575eaab5d22290 2024-11-15T22:13:06,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/c449399b7db04829b11f4387138cce07 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/c449399b7db04829b11f4387138cce07 2024-11-15T22:13:06,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/c449399b7db04829b11f4387138cce07, entries=200, sequenceid=122, filesize=38.6 K 2024-11-15T22:13:06,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/f726a790cf1b438b9b9df990bca620ab as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/f726a790cf1b438b9b9df990bca620ab 2024-11-15T22:13:06,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/f726a790cf1b438b9b9df990bca620ab, entries=150, sequenceid=122, filesize=11.7 K 2024-11-15T22:13:06,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/9a6d6da6ea154adf90575eaab5d22290 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/9a6d6da6ea154adf90575eaab5d22290 2024-11-15T22:13:06,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/9a6d6da6ea154adf90575eaab5d22290, entries=150, sequenceid=122, filesize=11.7 K 2024-11-15T22:13:06,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T22:13:06,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:06,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:06,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:06,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:06,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:06,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:06,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2a98cc30238e089ac8df02a2f0745572 in 627ms, sequenceid=122, compaction requested=false 2024-11-15T22:13:06,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:06,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:06,863 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:06,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T22:13:06,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115486a8d32b3b84ed99e2cf6da60e53d4c_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708786218/Put/seqid=0 2024-11-15T22:13:06,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742195_1371 (size=14794) 2024-11-15T22:13:06,916 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:06,928 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115486a8d32b3b84ed99e2cf6da60e53d4c_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115486a8d32b3b84ed99e2cf6da60e53d4c_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:06,929 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/7a437a4d5e1b45c995c99086475e439f, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:06,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/7a437a4d5e1b45c995c99086475e439f is 175, key is test_row_0/A:col10/1731708786218/Put/seqid=0 2024-11-15T22:13:06,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742196_1372 (size=39749) 2024-11-15T22:13:06,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T22:13:06,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:06,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:06,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:06,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:06,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:06,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708846942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:06,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:06,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708846949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:06,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:06,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708846955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:06,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:06,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708846957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:06,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708846958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T22:13:07,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708847066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708847066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708847077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708847077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708847077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T22:13:07,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:07,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:07,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:07,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:07,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:07,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T22:13:07,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:07,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:07,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:07,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:07,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:07,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708847287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708847287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708847287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708847287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708847287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,342 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/7a437a4d5e1b45c995c99086475e439f 2024-11-15T22:13:07,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/127fc688710347dcbf901ca6258b5862 is 50, key is test_row_0/B:col10/1731708786218/Put/seqid=0 2024-11-15T22:13:07,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742197_1373 (size=12151) 2024-11-15T22:13:07,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/127fc688710347dcbf901ca6258b5862 2024-11-15T22:13:07,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/303af36a57b644d6a9e1cc01525dd9e3 is 50, key is test_row_0/C:col10/1731708786218/Put/seqid=0 2024-11-15T22:13:07,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T22:13:07,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:07,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:07,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:07,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:07,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:07,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742198_1374 (size=12151) 2024-11-15T22:13:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:07,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/303af36a57b644d6a9e1cc01525dd9e3 2024-11-15T22:13:07,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/7a437a4d5e1b45c995c99086475e439f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/7a437a4d5e1b45c995c99086475e439f 2024-11-15T22:13:07,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/7a437a4d5e1b45c995c99086475e439f, entries=200, sequenceid=135, filesize=38.8 K 2024-11-15T22:13:07,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/127fc688710347dcbf901ca6258b5862 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/127fc688710347dcbf901ca6258b5862 2024-11-15T22:13:07,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/127fc688710347dcbf901ca6258b5862, entries=150, sequenceid=135, filesize=11.9 K 2024-11-15T22:13:07,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/303af36a57b644d6a9e1cc01525dd9e3 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/303af36a57b644d6a9e1cc01525dd9e3 2024-11-15T22:13:07,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/303af36a57b644d6a9e1cc01525dd9e3, entries=150, sequenceid=135, filesize=11.9 K 2024-11-15T22:13:07,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 2a98cc30238e089ac8df02a2f0745572 in 583ms, sequenceid=135, compaction requested=true 2024-11-15T22:13:07,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:07,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:13:07,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:07,446 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:07,446 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:07,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:13:07,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:07,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:13:07,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:07,447 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:07,447 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/A is initiating minor compaction (all files) 2024-11-15T22:13:07,447 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/A in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:07,447 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/daadc4a0f270438e939f3c9b0eec71a7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/c449399b7db04829b11f4387138cce07, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/7a437a4d5e1b45c995c99086475e439f] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=107.9 K 2024-11-15T22:13:07,447 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:07,448 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/daadc4a0f270438e939f3c9b0eec71a7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/c449399b7db04829b11f4387138cce07, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/7a437a4d5e1b45c995c99086475e439f] 2024-11-15T22:13:07,448 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:07,448 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting daadc4a0f270438e939f3c9b0eec71a7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731708785020 2024-11-15T22:13:07,448 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/B is initiating minor compaction (all files) 2024-11-15T22:13:07,448 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/B in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:07,448 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/b95fb6097065424ba808150b6be0d294, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/f726a790cf1b438b9b9df990bca620ab, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/127fc688710347dcbf901ca6258b5862] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=35.5 K 2024-11-15T22:13:07,448 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting c449399b7db04829b11f4387138cce07, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731708785821 2024-11-15T22:13:07,448 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting b95fb6097065424ba808150b6be0d294, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731708785020 2024-11-15T22:13:07,449 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting f726a790cf1b438b9b9df990bca620ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731708785821 2024-11-15T22:13:07,449 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a437a4d5e1b45c995c99086475e439f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731708786203 2024-11-15T22:13:07,449 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 127fc688710347dcbf901ca6258b5862, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731708786208 2024-11-15T22:13:07,459 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:07,470 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115960349ef6cd1475eb03fb0e9b3b871c9_2a98cc30238e089ac8df02a2f0745572 store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:07,471 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#B#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:07,472 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/37d901232e69424fa136c009f09ac9b3 is 50, key is test_row_0/B:col10/1731708786218/Put/seqid=0 2024-11-15T22:13:07,472 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115960349ef6cd1475eb03fb0e9b3b871c9_2a98cc30238e089ac8df02a2f0745572, store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:07,472 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115960349ef6cd1475eb03fb0e9b3b871c9_2a98cc30238e089ac8df02a2f0745572 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:07,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742199_1375 (size=4469) 2024-11-15T22:13:07,483 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#A#compaction#311 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:07,483 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/8876c9713b0b43f0919fe5501ecd364f is 175, key is test_row_0/A:col10/1731708786218/Put/seqid=0 2024-11-15T22:13:07,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742200_1376 (size=12459) 2024-11-15T22:13:07,504 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/37d901232e69424fa136c009f09ac9b3 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/37d901232e69424fa136c009f09ac9b3 2024-11-15T22:13:07,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742201_1377 (size=31413) 2024-11-15T22:13:07,515 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/8876c9713b0b43f0919fe5501ecd364f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/8876c9713b0b43f0919fe5501ecd364f 2024-11-15T22:13:07,519 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/B of 2a98cc30238e089ac8df02a2f0745572 into 37d901232e69424fa136c009f09ac9b3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:07,519 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:07,520 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/B, priority=13, startTime=1731708787446; duration=0sec 2024-11-15T22:13:07,520 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:07,520 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:B 2024-11-15T22:13:07,520 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:07,522 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:07,522 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/C is initiating minor compaction (all files) 2024-11-15T22:13:07,523 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/C in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:07,523 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/6121a0b8c97443e4a10bd7a2397cf401, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/9a6d6da6ea154adf90575eaab5d22290, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/303af36a57b644d6a9e1cc01525dd9e3] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=35.5 K 2024-11-15T22:13:07,523 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 6121a0b8c97443e4a10bd7a2397cf401, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731708785020 2024-11-15T22:13:07,523 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/A of 2a98cc30238e089ac8df02a2f0745572 into 8876c9713b0b43f0919fe5501ecd364f(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:07,523 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:07,523 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/A, priority=13, startTime=1731708787446; duration=0sec 2024-11-15T22:13:07,524 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:07,524 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:A 2024-11-15T22:13:07,524 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a6d6da6ea154adf90575eaab5d22290, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731708785821 2024-11-15T22:13:07,524 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 303af36a57b644d6a9e1cc01525dd9e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731708786208 2024-11-15T22:13:07,533 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#C#compaction#313 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:07,534 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/7f8167b87f6d449588fd1c03f93d9275 is 50, key is test_row_0/C:col10/1731708786218/Put/seqid=0 2024-11-15T22:13:07,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742202_1378 (size=12459) 2024-11-15T22:13:07,565 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/7f8167b87f6d449588fd1c03f93d9275 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7f8167b87f6d449588fd1c03f93d9275 2024-11-15T22:13:07,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T22:13:07,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:07,571 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T22:13:07,573 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/C of 2a98cc30238e089ac8df02a2f0745572 into 7f8167b87f6d449588fd1c03f93d9275(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:07,573 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:07,574 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/C, priority=13, startTime=1731708787447; duration=0sec 2024-11-15T22:13:07,574 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:07,574 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:C 2024-11-15T22:13:07,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cbcbdd000f1d48d8bf3f2eeb8dd4ca0d_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708786955/Put/seqid=0 2024-11-15T22:13:07,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742203_1379 (size=12304) 2024-11-15T22:13:07,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:07,601 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cbcbdd000f1d48d8bf3f2eeb8dd4ca0d_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cbcbdd000f1d48d8bf3f2eeb8dd4ca0d_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:07,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/6743620a97ad406bb6e352b3bbfe9539, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:07,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/6743620a97ad406bb6e352b3bbfe9539 is 175, key is test_row_0/A:col10/1731708786955/Put/seqid=0 2024-11-15T22:13:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:07,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:07,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742204_1380 (size=31105) 2024-11-15T22:13:07,626 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=163, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/6743620a97ad406bb6e352b3bbfe9539 2024-11-15T22:13:07,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708847618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708847618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708847625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708847627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708847631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/b077008ac27846788282c5deb7b52544 is 50, key is test_row_0/B:col10/1731708786955/Put/seqid=0 2024-11-15T22:13:07,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742205_1381 (size=12151) 2024-11-15T22:13:07,659 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/b077008ac27846788282c5deb7b52544 2024-11-15T22:13:07,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/3d20ca780c8045d086ee849e33f05682 is 50, key is test_row_0/C:col10/1731708786955/Put/seqid=0 2024-11-15T22:13:07,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742206_1382 (size=12151) 2024-11-15T22:13:07,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708847736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708847736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708847747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708847747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708847747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708847957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708847957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708847970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708847973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:07,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708847973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:07,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:07,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:07,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T22:13:08,083 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/3d20ca780c8045d086ee849e33f05682 2024-11-15T22:13:08,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/6743620a97ad406bb6e352b3bbfe9539 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/6743620a97ad406bb6e352b3bbfe9539 2024-11-15T22:13:08,093 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/6743620a97ad406bb6e352b3bbfe9539, entries=150, sequenceid=163, filesize=30.4 K 2024-11-15T22:13:08,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/b077008ac27846788282c5deb7b52544 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/b077008ac27846788282c5deb7b52544 2024-11-15T22:13:08,101 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/b077008ac27846788282c5deb7b52544, entries=150, sequenceid=163, filesize=11.9 K 2024-11-15T22:13:08,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/3d20ca780c8045d086ee849e33f05682 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/3d20ca780c8045d086ee849e33f05682 2024-11-15T22:13:08,106 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/3d20ca780c8045d086ee849e33f05682, entries=150, sequenceid=163, filesize=11.9 K 2024-11-15T22:13:08,108 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2a98cc30238e089ac8df02a2f0745572 in 537ms, sequenceid=163, compaction requested=false 2024-11-15T22:13:08,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:08,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:08,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-15T22:13:08,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-15T22:13:08,112 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-15T22:13:08,112 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2740 sec 2024-11-15T22:13:08,115 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.2800 sec 2024-11-15T22:13:08,287 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:08,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T22:13:08,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:08,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d6ba9c92d5e34723b9763a3a0039339d_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708787612/Put/seqid=0 2024-11-15T22:13:08,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742207_1383 (size=14794) 2024-11-15T22:13:08,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708848338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:08,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708848349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:08,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708848353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:08,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708848359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:08,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:08,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708848360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:08,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:08,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:08,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:08,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:08,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708848467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:08,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708848467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:08,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:08,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:08,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:08,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708848486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:08,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:08,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708848487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:08,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:08,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708848487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:08,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:08,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708848687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:08,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708848687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:08,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:08,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708848707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:08,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:08,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:08,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708848708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:08,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:08,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:08,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708848708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:08,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:08,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:08,740 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:08,750 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d6ba9c92d5e34723b9763a3a0039339d_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d6ba9c92d5e34723b9763a3a0039339d_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:08,752 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/1215a3de8bb04d47b7253366ac13c142, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:08,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/1215a3de8bb04d47b7253366ac13c142 is 175, key is test_row_0/A:col10/1731708787612/Put/seqid=0 2024-11-15T22:13:08,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742208_1384 (size=39749) 2024-11-15T22:13:09,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708849007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:09,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:09,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708849007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:09,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:09,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:09,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:09,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708849030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:09,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:09,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:09,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:09,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708849030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:09,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708849030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:09,175 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/1215a3de8bb04d47b7253366ac13c142 2024-11-15T22:13:09,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/5e69b604c26440d4bfaa8684039ca6fb is 50, key is test_row_0/B:col10/1731708787612/Put/seqid=0 2024-11-15T22:13:09,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742209_1385 (size=12151) 2024-11-15T22:13:09,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/5e69b604c26440d4bfaa8684039ca6fb 2024-11-15T22:13:09,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/d270dd3bf07f43388b3d15fe1fe21234 is 50, key is test_row_0/C:col10/1731708787612/Put/seqid=0 2024-11-15T22:13:09,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742210_1386 (size=12151) 2024-11-15T22:13:09,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708849526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:09,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:09,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708849526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:09,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:09,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:09,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:09,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:09,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:09,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:09,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708849547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:09,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:09,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708849547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:09,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:09,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:09,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708849547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:09,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:09,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:09,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:09,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/d270dd3bf07f43388b3d15fe1fe21234 2024-11-15T22:13:09,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/1215a3de8bb04d47b7253366ac13c142 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/1215a3de8bb04d47b7253366ac13c142 2024-11-15T22:13:09,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/1215a3de8bb04d47b7253366ac13c142, entries=200, sequenceid=175, filesize=38.8 K 2024-11-15T22:13:09,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/5e69b604c26440d4bfaa8684039ca6fb as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/5e69b604c26440d4bfaa8684039ca6fb 2024-11-15T22:13:09,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/5e69b604c26440d4bfaa8684039ca6fb, entries=150, sequenceid=175, filesize=11.9 K 2024-11-15T22:13:09,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/d270dd3bf07f43388b3d15fe1fe21234 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/d270dd3bf07f43388b3d15fe1fe21234 2024-11-15T22:13:09,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/d270dd3bf07f43388b3d15fe1fe21234, entries=150, sequenceid=175, filesize=11.9 K 2024-11-15T22:13:09,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 2a98cc30238e089ac8df02a2f0745572 in 1400ms, sequenceid=175, compaction requested=true 2024-11-15T22:13:09,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:09,688 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:09,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:13:09,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:09,688 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:09,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:13:09,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:09,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:13:09,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:09,692 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:09,692 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:09,692 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/A is initiating minor compaction (all files) 2024-11-15T22:13:09,692 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/B is initiating minor compaction (all files) 2024-11-15T22:13:09,692 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/A in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:09,692 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/8876c9713b0b43f0919fe5501ecd364f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/6743620a97ad406bb6e352b3bbfe9539, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/1215a3de8bb04d47b7253366ac13c142] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=99.9 K 2024-11-15T22:13:09,692 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:09,692 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/8876c9713b0b43f0919fe5501ecd364f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/6743620a97ad406bb6e352b3bbfe9539, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/1215a3de8bb04d47b7253366ac13c142] 2024-11-15T22:13:09,692 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/B in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:09,692 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/37d901232e69424fa136c009f09ac9b3, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/b077008ac27846788282c5deb7b52544, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/5e69b604c26440d4bfaa8684039ca6fb] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=35.9 K 2024-11-15T22:13:09,693 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 8876c9713b0b43f0919fe5501ecd364f, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731708786208 2024-11-15T22:13:09,693 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37d901232e69424fa136c009f09ac9b3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731708786208 2024-11-15T22:13:09,693 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 6743620a97ad406bb6e352b3bbfe9539, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731708786953 2024-11-15T22:13:09,693 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting b077008ac27846788282c5deb7b52544, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731708786953 2024-11-15T22:13:09,694 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e69b604c26440d4bfaa8684039ca6fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731708787612 2024-11-15T22:13:09,694 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 1215a3de8bb04d47b7253366ac13c142, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731708787612 2024-11-15T22:13:09,715 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#B#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:09,716 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/77ced7c49f5e4d43bc9c1c0a6764699f is 50, key is test_row_0/B:col10/1731708787612/Put/seqid=0 2024-11-15T22:13:09,717 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:09,734 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115fba2de5f21794a2c9d4a3a33295d04cb_2a98cc30238e089ac8df02a2f0745572 store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:09,736 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115fba2de5f21794a2c9d4a3a33295d04cb_2a98cc30238e089ac8df02a2f0745572, store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:09,736 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115fba2de5f21794a2c9d4a3a33295d04cb_2a98cc30238e089ac8df02a2f0745572 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:09,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742211_1387 (size=12561) 2024-11-15T22:13:09,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742212_1388 (size=4469) 2024-11-15T22:13:09,762 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#A#compaction#321 average throughput is 0.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:09,762 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/46c1be5e9f934f53ae40879068d40b47 is 175, key is test_row_0/A:col10/1731708787612/Put/seqid=0 2024-11-15T22:13:09,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742213_1389 (size=31515) 2024-11-15T22:13:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T22:13:10,007 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:10,007 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:13:10,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-15T22:13:10,009 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:13:10,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T22:13:10,010 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:13:10,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:13:10,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T22:13:10,162 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/77ced7c49f5e4d43bc9c1c0a6764699f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/77ced7c49f5e4d43bc9c1c0a6764699f 2024-11-15T22:13:10,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T22:13:10,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:10,163 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T22:13:10,171 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/B of 2a98cc30238e089ac8df02a2f0745572 into 77ced7c49f5e4d43bc9c1c0a6764699f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:10,171 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:10,171 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/B, priority=13, startTime=1731708789688; duration=0sec 2024-11-15T22:13:10,171 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:10,171 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:B 2024-11-15T22:13:10,171 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:10,173 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:10,173 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/C is initiating minor compaction (all files) 2024-11-15T22:13:10,173 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/C in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:10,173 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7f8167b87f6d449588fd1c03f93d9275, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/3d20ca780c8045d086ee849e33f05682, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/d270dd3bf07f43388b3d15fe1fe21234] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=35.9 K 2024-11-15T22:13:10,175 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f8167b87f6d449588fd1c03f93d9275, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731708786208 2024-11-15T22:13:10,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115343f49de939345e4937fa7f30f8119c0_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708788335/Put/seqid=0 2024-11-15T22:13:10,177 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d20ca780c8045d086ee849e33f05682, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731708786953 2024-11-15T22:13:10,178 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/46c1be5e9f934f53ae40879068d40b47 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/46c1be5e9f934f53ae40879068d40b47 2024-11-15T22:13:10,178 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting d270dd3bf07f43388b3d15fe1fe21234, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731708787612 2024-11-15T22:13:10,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742214_1390 (size=12304) 2024-11-15T22:13:10,185 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/A of 2a98cc30238e089ac8df02a2f0745572 into 46c1be5e9f934f53ae40879068d40b47(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:10,185 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:10,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:10,186 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/A, priority=13, startTime=1731708789687; duration=0sec 2024-11-15T22:13:10,186 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:10,186 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:A 2024-11-15T22:13:10,190 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#C#compaction#323 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:10,190 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/d8ef443458bc4a60aef77cdccde15a3e is 50, key is test_row_0/C:col10/1731708787612/Put/seqid=0 2024-11-15T22:13:10,194 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115343f49de939345e4937fa7f30f8119c0_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115343f49de939345e4937fa7f30f8119c0_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:10,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/90c3ee31a12b4b2e8a73ff641732ad8d, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:10,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/90c3ee31a12b4b2e8a73ff641732ad8d is 175, key is test_row_0/A:col10/1731708788335/Put/seqid=0 2024-11-15T22:13:10,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742215_1391 (size=12561) 2024-11-15T22:13:10,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742216_1392 (size=31105) 2024-11-15T22:13:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T22:13:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:10,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:10,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708850566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:10,573 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_0' in 2a98cc30238e089ac8df02a2f0745572 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:13:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:10,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:10,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708850568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708850571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:10,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708850571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:10,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708850572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:10,622 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/90c3ee31a12b4b2e8a73ff641732ad8d 2024-11-15T22:13:10,624 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/d8ef443458bc4a60aef77cdccde15a3e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/d8ef443458bc4a60aef77cdccde15a3e 2024-11-15T22:13:10,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/3daac38e8b1a48b497422faa351893b2 is 50, key is test_row_0/B:col10/1731708788335/Put/seqid=0 2024-11-15T22:13:10,635 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/C of 2a98cc30238e089ac8df02a2f0745572 into d8ef443458bc4a60aef77cdccde15a3e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:10,635 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:10,635 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/C, priority=13, startTime=1731708789688; duration=0sec 2024-11-15T22:13:10,635 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:10,635 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:C 2024-11-15T22:13:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T22:13:10,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742217_1393 (size=12151) 2024-11-15T22:13:10,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:10,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708850687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:10,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:10,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:10,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708850687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:10,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708850687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708850687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:10,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:10,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708850906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:10,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708850907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:10,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708850907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:10,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708850907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:11,039 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/3daac38e8b1a48b497422faa351893b2 2024-11-15T22:13:11,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/14f042b2194f4ab5964e0e704a683bcc is 50, key is test_row_0/C:col10/1731708788335/Put/seqid=0 2024-11-15T22:13:11,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742218_1394 (size=12151) 2024-11-15T22:13:11,056 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/14f042b2194f4ab5964e0e704a683bcc 2024-11-15T22:13:11,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/90c3ee31a12b4b2e8a73ff641732ad8d as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/90c3ee31a12b4b2e8a73ff641732ad8d 2024-11-15T22:13:11,069 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/90c3ee31a12b4b2e8a73ff641732ad8d, entries=150, sequenceid=200, filesize=30.4 K 2024-11-15T22:13:11,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/3daac38e8b1a48b497422faa351893b2 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/3daac38e8b1a48b497422faa351893b2 2024-11-15T22:13:11,076 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/3daac38e8b1a48b497422faa351893b2, entries=150, sequenceid=200, filesize=11.9 K 2024-11-15T22:13:11,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/14f042b2194f4ab5964e0e704a683bcc as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/14f042b2194f4ab5964e0e704a683bcc 2024-11-15T22:13:11,084 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/14f042b2194f4ab5964e0e704a683bcc, entries=150, sequenceid=200, filesize=11.9 K 2024-11-15T22:13:11,085 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2a98cc30238e089ac8df02a2f0745572 in 922ms, sequenceid=200, compaction requested=false 2024-11-15T22:13:11,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:11,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:11,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-15T22:13:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-15T22:13:11,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-15T22:13:11,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0770 sec 2024-11-15T22:13:11,093 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.0830 sec 2024-11-15T22:13:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T22:13:11,147 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:11,147 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:13:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-15T22:13:11,150 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:13:11,151 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:13:11,151 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:13:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T22:13:11,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:11,232 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:11,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T22:13:11,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158c0392fdeabd4feea56458331728a331_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708790571/Put/seqid=0 2024-11-15T22:13:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T22:13:11,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742219_1395 (size=17284) 2024-11-15T22:13:11,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T22:13:11,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:11,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:11,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:11,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:11,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:11,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708851288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:11,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:11,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708851292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:11,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:11,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708851298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:11,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:11,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708851308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:11,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708851417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:11,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708851417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:11,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708851417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:11,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708851426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:11,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:11,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T22:13:11,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:11,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:11,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:11,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:11,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:11,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T22:13:11,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T22:13:11,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:11,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:11,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:11,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:11,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:11,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708851636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:11,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708851637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:11,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708851637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:11,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:11,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708851639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:11,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:11,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:11,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:11,670 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:11,675 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158c0392fdeabd4feea56458331728a331_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158c0392fdeabd4feea56458331728a331_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:11,678 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/918e1c647bca427ca1b99c1542131797, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:11,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/918e1c647bca427ca1b99c1542131797 is 175, key is test_row_0/A:col10/1731708790571/Put/seqid=0 2024-11-15T22:13:11,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742220_1396 (size=48389) 2024-11-15T22:13:11,683 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/918e1c647bca427ca1b99c1542131797 2024-11-15T22:13:11,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/15cdcc8cf3f84ae59dcda1aaff127be1 is 50, key is test_row_0/B:col10/1731708790571/Put/seqid=0 2024-11-15T22:13:11,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742221_1397 (size=12151) 2024-11-15T22:13:11,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/15cdcc8cf3f84ae59dcda1aaff127be1 2024-11-15T22:13:11,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/e4f1d19b388349c1ba267aadddf291a6 is 50, key is test_row_0/C:col10/1731708790571/Put/seqid=0 2024-11-15T22:13:11,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742222_1398 (size=12151) 2024-11-15T22:13:11,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/e4f1d19b388349c1ba267aadddf291a6 2024-11-15T22:13:11,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/918e1c647bca427ca1b99c1542131797 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/918e1c647bca427ca1b99c1542131797 2024-11-15T22:13:11,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T22:13:11,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:11,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:11,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:11,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:11,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:11,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:11,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/918e1c647bca427ca1b99c1542131797, entries=250, sequenceid=216, filesize=47.3 K 2024-11-15T22:13:11,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/15cdcc8cf3f84ae59dcda1aaff127be1 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/15cdcc8cf3f84ae59dcda1aaff127be1 2024-11-15T22:13:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T22:13:11,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/15cdcc8cf3f84ae59dcda1aaff127be1, entries=150, sequenceid=216, filesize=11.9 K 2024-11-15T22:13:11,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/e4f1d19b388349c1ba267aadddf291a6 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e4f1d19b388349c1ba267aadddf291a6 2024-11-15T22:13:11,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e4f1d19b388349c1ba267aadddf291a6, entries=150, sequenceid=216, filesize=11.9 K 2024-11-15T22:13:11,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 2a98cc30238e089ac8df02a2f0745572 in 553ms, sequenceid=216, compaction requested=true 2024-11-15T22:13:11,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:11,785 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:11,786 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:11,786 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/A is initiating minor compaction (all files) 2024-11-15T22:13:11,786 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/A in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:11,786 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/46c1be5e9f934f53ae40879068d40b47, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/90c3ee31a12b4b2e8a73ff641732ad8d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/918e1c647bca427ca1b99c1542131797] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=108.4 K 2024-11-15T22:13:11,786 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:11,786 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/46c1be5e9f934f53ae40879068d40b47, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/90c3ee31a12b4b2e8a73ff641732ad8d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/918e1c647bca427ca1b99c1542131797] 2024-11-15T22:13:11,787 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46c1be5e9f934f53ae40879068d40b47, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731708787612 2024-11-15T22:13:11,787 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90c3ee31a12b4b2e8a73ff641732ad8d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731708788326 2024-11-15T22:13:11,787 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 918e1c647bca427ca1b99c1542131797, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731708790558 2024-11-15T22:13:11,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:13:11,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:11,792 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:11,794 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:11,794 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/B is initiating minor compaction (all files) 2024-11-15T22:13:11,794 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/B in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:11,794 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/77ced7c49f5e4d43bc9c1c0a6764699f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/3daac38e8b1a48b497422faa351893b2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/15cdcc8cf3f84ae59dcda1aaff127be1] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=36.0 K 2024-11-15T22:13:11,794 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 77ced7c49f5e4d43bc9c1c0a6764699f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731708787612 2024-11-15T22:13:11,795 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 3daac38e8b1a48b497422faa351893b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731708788326 2024-11-15T22:13:11,795 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 15cdcc8cf3f84ae59dcda1aaff127be1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731708790565 2024-11-15T22:13:11,797 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:11,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:13:11,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:11,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:13:11,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:11,801 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115259654ee4f07468dbd598d36ac78c827_2a98cc30238e089ac8df02a2f0745572 store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:11,803 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#B#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:11,803 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115259654ee4f07468dbd598d36ac78c827_2a98cc30238e089ac8df02a2f0745572, store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:11,803 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115259654ee4f07468dbd598d36ac78c827_2a98cc30238e089ac8df02a2f0745572 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:11,804 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/9b2037fe74904ece884b1ecee4cbd4de is 50, key is test_row_0/B:col10/1731708790571/Put/seqid=0 2024-11-15T22:13:11,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742224_1400 (size=12663) 2024-11-15T22:13:11,836 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/9b2037fe74904ece884b1ecee4cbd4de as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/9b2037fe74904ece884b1ecee4cbd4de 2024-11-15T22:13:11,840 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/B of 2a98cc30238e089ac8df02a2f0745572 into 9b2037fe74904ece884b1ecee4cbd4de(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:11,840 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:11,840 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/B, priority=13, startTime=1731708791792; duration=0sec 2024-11-15T22:13:11,841 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:11,841 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:B 2024-11-15T22:13:11,841 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:11,842 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:11,842 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/C is initiating minor compaction (all files) 2024-11-15T22:13:11,842 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/C in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:11,842 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/d8ef443458bc4a60aef77cdccde15a3e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/14f042b2194f4ab5964e0e704a683bcc, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e4f1d19b388349c1ba267aadddf291a6] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=36.0 K 2024-11-15T22:13:11,843 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting d8ef443458bc4a60aef77cdccde15a3e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731708787612 2024-11-15T22:13:11,843 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 14f042b2194f4ab5964e0e704a683bcc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731708788326 2024-11-15T22:13:11,843 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting e4f1d19b388349c1ba267aadddf291a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731708790565 2024-11-15T22:13:11,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742223_1399 (size=4469) 2024-11-15T22:13:11,860 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#A#compaction#329 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:11,860 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/ca28651442a74208933d4a233ac33ede is 175, key is test_row_0/A:col10/1731708790571/Put/seqid=0 2024-11-15T22:13:11,863 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#C#compaction#331 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:11,863 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/7f777af3c64d440898944807430f1197 is 50, key is test_row_0/C:col10/1731708790571/Put/seqid=0 2024-11-15T22:13:11,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742225_1401 (size=31617) 2024-11-15T22:13:11,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T22:13:11,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:11,924 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T22:13:11,932 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/ca28651442a74208933d4a233ac33ede as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/ca28651442a74208933d4a233ac33ede 2024-11-15T22:13:11,938 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/A of 2a98cc30238e089ac8df02a2f0745572 into ca28651442a74208933d4a233ac33ede(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:11,938 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:11,938 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/A, priority=13, startTime=1731708791785; duration=0sec 2024-11-15T22:13:11,939 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:11,939 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:A 2024-11-15T22:13:11,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742226_1402 (size=12663) 2024-11-15T22:13:11,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111537c408ac7fbd456eb24207a59f08428c_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708791295/Put/seqid=0 2024-11-15T22:13:11,957 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/7f777af3c64d440898944807430f1197 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7f777af3c64d440898944807430f1197 2024-11-15T22:13:11,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742227_1403 (size=12304) 2024-11-15T22:13:11,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:11,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:11,976 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111537c408ac7fbd456eb24207a59f08428c_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111537c408ac7fbd456eb24207a59f08428c_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:11,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/5b34f975dd344157b986074777afee54, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:11,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/5b34f975dd344157b986074777afee54 is 175, key is test_row_0/A:col10/1731708791295/Put/seqid=0 2024-11-15T22:13:11,981 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/C of 2a98cc30238e089ac8df02a2f0745572 into 7f777af3c64d440898944807430f1197(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:11,982 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:11,982 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/C, priority=13, startTime=1731708791797; duration=0sec 2024-11-15T22:13:11,982 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:11,982 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:C 2024-11-15T22:13:11,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742228_1404 (size=31105) 2024-11-15T22:13:11,991 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/5b34f975dd344157b986074777afee54 2024-11-15T22:13:12,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/55957fbe4f534852803282614c5ee85c is 50, key is test_row_0/B:col10/1731708791295/Put/seqid=0 2024-11-15T22:13:12,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742229_1405 (size=12151) 2024-11-15T22:13:12,018 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/55957fbe4f534852803282614c5ee85c 2024-11-15T22:13:12,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/d9c39ad10cc347b083c9cdd758f403c4 is 50, key is test_row_0/C:col10/1731708791295/Put/seqid=0 2024-11-15T22:13:12,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708852008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:12,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708852066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708852068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708852068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:12,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742230_1406 (size=12151) 2024-11-15T22:13:12,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708852167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:12,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708852186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708852186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:12,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:12,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708852186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:12,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T22:13:12,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708852387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:12,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708852397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708852397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:12,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708852406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:12,490 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/d9c39ad10cc347b083c9cdd758f403c4 2024-11-15T22:13:12,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/5b34f975dd344157b986074777afee54 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/5b34f975dd344157b986074777afee54 2024-11-15T22:13:12,505 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/5b34f975dd344157b986074777afee54, entries=150, sequenceid=240, filesize=30.4 K 2024-11-15T22:13:12,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/55957fbe4f534852803282614c5ee85c as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/55957fbe4f534852803282614c5ee85c 2024-11-15T22:13:12,510 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/55957fbe4f534852803282614c5ee85c, entries=150, sequenceid=240, filesize=11.9 K 2024-11-15T22:13:12,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/d9c39ad10cc347b083c9cdd758f403c4 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/d9c39ad10cc347b083c9cdd758f403c4 2024-11-15T22:13:12,516 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/d9c39ad10cc347b083c9cdd758f403c4, entries=150, sequenceid=240, filesize=11.9 K 2024-11-15T22:13:12,518 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 2a98cc30238e089ac8df02a2f0745572 in 595ms, sequenceid=240, compaction requested=false 2024-11-15T22:13:12,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:12,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:12,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-15T22:13:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-15T22:13:12,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-15T22:13:12,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3700 sec 2024-11-15T22:13:12,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.3760 sec 2024-11-15T22:13:12,593 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:12,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T22:13:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:12,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111531ea8eafe15d405d8d95dfef25464d29_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708792591/Put/seqid=0 2024-11-15T22:13:12,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742231_1407 (size=14794) 2024-11-15T22:13:12,609 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:12,613 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111531ea8eafe15d405d8d95dfef25464d29_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111531ea8eafe15d405d8d95dfef25464d29_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:12,614 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/b9047a12afc04d1ebf904bfe7eda5267, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:12,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/b9047a12afc04d1ebf904bfe7eda5267 is 175, key is test_row_0/A:col10/1731708792591/Put/seqid=0 2024-11-15T22:13:12,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742232_1408 (size=39749) 2024-11-15T22:13:12,625 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/b9047a12afc04d1ebf904bfe7eda5267 2024-11-15T22:13:12,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/74c0ff4f13ae4720ac16aeece9d42307 is 50, key is test_row_0/B:col10/1731708792591/Put/seqid=0 2024-11-15T22:13:12,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742233_1409 (size=12151) 2024-11-15T22:13:12,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708852729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:12,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708852732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:12,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708852733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708852735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708852736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:12,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708852846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708852846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:12,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708852856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708852856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:12,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708852867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:12,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:12,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:13,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/74c0ff4f13ae4720ac16aeece9d42307 2024-11-15T22:13:13,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/7ffa78386e2b43e0866770473643f49b is 50, key is test_row_0/C:col10/1731708792591/Put/seqid=0 2024-11-15T22:13:13,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742234_1410 (size=12151) 2024-11-15T22:13:13,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:13,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708853056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:13,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:13,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708853056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:13,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:13,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708853077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:13,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:13,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708853077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:13,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:13,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708853087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:13,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T22:13:13,297 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:13,298 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:13:13,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-15T22:13:13,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T22:13:13,301 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:13:13,302 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:13:13,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:13:13,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708853376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:13,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:13,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:13,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708853377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:13,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:13,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708853387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708853387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:13,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:13,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T22:13:13,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708853407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:13,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:13,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/7ffa78386e2b43e0866770473643f49b 2024-11-15T22:13:13,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T22:13:13,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:13,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:13,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:13,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:13,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:13,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/b9047a12afc04d1ebf904bfe7eda5267 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/b9047a12afc04d1ebf904bfe7eda5267 2024-11-15T22:13:13,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/b9047a12afc04d1ebf904bfe7eda5267, entries=200, sequenceid=256, filesize=38.8 K 2024-11-15T22:13:13,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/74c0ff4f13ae4720ac16aeece9d42307 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/74c0ff4f13ae4720ac16aeece9d42307 2024-11-15T22:13:13,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/74c0ff4f13ae4720ac16aeece9d42307, entries=150, sequenceid=256, filesize=11.9 K 2024-11-15T22:13:13,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/7ffa78386e2b43e0866770473643f49b as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7ffa78386e2b43e0866770473643f49b 2024-11-15T22:13:13,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7ffa78386e2b43e0866770473643f49b, entries=150, sequenceid=256, filesize=11.9 K 2024-11-15T22:13:13,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 2a98cc30238e089ac8df02a2f0745572 in 904ms, sequenceid=256, compaction requested=true 2024-11-15T22:13:13,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:13,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:13:13,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:13,497 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:13,497 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:13,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:13:13,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:13,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:13:13,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:13,499 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:13,499 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/A is initiating minor compaction (all files) 2024-11-15T22:13:13,499 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/A in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:13,499 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/ca28651442a74208933d4a233ac33ede, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/5b34f975dd344157b986074777afee54, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/b9047a12afc04d1ebf904bfe7eda5267] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=100.1 K 2024-11-15T22:13:13,499 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:13,499 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/ca28651442a74208933d4a233ac33ede, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/5b34f975dd344157b986074777afee54, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/b9047a12afc04d1ebf904bfe7eda5267] 2024-11-15T22:13:13,500 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:13,500 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/B is initiating minor compaction (all files) 2024-11-15T22:13:13,500 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/B in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:13,500 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/9b2037fe74904ece884b1ecee4cbd4de, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/55957fbe4f534852803282614c5ee85c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/74c0ff4f13ae4720ac16aeece9d42307] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=36.1 K 2024-11-15T22:13:13,500 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca28651442a74208933d4a233ac33ede, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731708790565 2024-11-15T22:13:13,500 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b34f975dd344157b986074777afee54, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731708791283 2024-11-15T22:13:13,500 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b2037fe74904ece884b1ecee4cbd4de, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731708790565 2024-11-15T22:13:13,501 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9047a12afc04d1ebf904bfe7eda5267, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731708791966 2024-11-15T22:13:13,501 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 55957fbe4f534852803282614c5ee85c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731708791283 2024-11-15T22:13:13,501 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 74c0ff4f13ae4720ac16aeece9d42307, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731708791966 2024-11-15T22:13:13,510 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#B#compaction#338 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:13,511 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/28deb18a707249f2bf707490b8d9f38f is 50, key is test_row_0/B:col10/1731708792591/Put/seqid=0 2024-11-15T22:13:13,514 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:13,518 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111539dc26d5ab94453e92039192efce57d5_2a98cc30238e089ac8df02a2f0745572 store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:13,520 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111539dc26d5ab94453e92039192efce57d5_2a98cc30238e089ac8df02a2f0745572, store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:13,520 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111539dc26d5ab94453e92039192efce57d5_2a98cc30238e089ac8df02a2f0745572 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:13,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742235_1411 (size=12765) 2024-11-15T22:13:13,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742236_1412 (size=4469) 2024-11-15T22:13:13,540 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#A#compaction#339 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:13,541 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/08f85d600002405492862852a85e7941 is 175, key is test_row_0/A:col10/1731708792591/Put/seqid=0 2024-11-15T22:13:13,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742237_1413 (size=31719) 2024-11-15T22:13:13,555 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/08f85d600002405492862852a85e7941 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/08f85d600002405492862852a85e7941 2024-11-15T22:13:13,563 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/A of 2a98cc30238e089ac8df02a2f0745572 into 08f85d600002405492862852a85e7941(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:13,563 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:13,563 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/A, priority=13, startTime=1731708793497; duration=0sec 2024-11-15T22:13:13,563 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:13,563 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:A 2024-11-15T22:13:13,563 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:13,564 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:13,564 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/C is initiating minor compaction (all files) 2024-11-15T22:13:13,564 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/C in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:13,564 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7f777af3c64d440898944807430f1197, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/d9c39ad10cc347b083c9cdd758f403c4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7ffa78386e2b43e0866770473643f49b] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=36.1 K 2024-11-15T22:13:13,565 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f777af3c64d440898944807430f1197, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731708790565 2024-11-15T22:13:13,565 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9c39ad10cc347b083c9cdd758f403c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731708791283 2024-11-15T22:13:13,566 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ffa78386e2b43e0866770473643f49b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731708791966 2024-11-15T22:13:13,572 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#C#compaction#340 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:13,573 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/3cd3f793587f4b579cba6c04184cf436 is 50, key is test_row_0/C:col10/1731708792591/Put/seqid=0 2024-11-15T22:13:13,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742238_1414 (size=12765) 2024-11-15T22:13:13,586 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/3cd3f793587f4b579cba6c04184cf436 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/3cd3f793587f4b579cba6c04184cf436 2024-11-15T22:13:13,593 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/C of 2a98cc30238e089ac8df02a2f0745572 into 3cd3f793587f4b579cba6c04184cf436(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:13,593 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:13,593 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/C, priority=13, startTime=1731708793497; duration=0sec 2024-11-15T22:13:13,594 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:13,594 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:C 2024-11-15T22:13:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T22:13:13,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T22:13:13,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:13,618 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T22:13:13,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c7a0725a67b14fe59a26f2b657e54de9_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708792717/Put/seqid=0 2024-11-15T22:13:13,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742239_1415 (size=12454) 2024-11-15T22:13:13,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:13,664 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c7a0725a67b14fe59a26f2b657e54de9_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c7a0725a67b14fe59a26f2b657e54de9_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:13,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/3ae6cd7eda7c4405905894576df43bd5, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:13,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/3ae6cd7eda7c4405905894576df43bd5 is 175, key is test_row_0/A:col10/1731708792717/Put/seqid=0 2024-11-15T22:13:13,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742240_1416 (size=31255) 2024-11-15T22:13:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:13,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:13,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T22:13:13,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708853923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:13,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:13,941 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/28deb18a707249f2bf707490b8d9f38f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/28deb18a707249f2bf707490b8d9f38f 2024-11-15T22:13:13,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708853932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:13,946 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/B of 2a98cc30238e089ac8df02a2f0745572 into 28deb18a707249f2bf707490b8d9f38f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:13,946 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:13,946 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/B, priority=13, startTime=1731708793497; duration=0sec 2024-11-15T22:13:13,946 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:13,946 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:B 2024-11-15T22:13:13,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:13,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708853933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:13,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:13,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708853935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:13,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:13,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708853936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:13,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:13,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:13,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:13,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:14,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708854047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708854047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:14,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708854056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708854056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708854056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:14,080 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/3ae6cd7eda7c4405905894576df43bd5 2024-11-15T22:13:14,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/535d06ad203b44eb987657a631f39786 is 50, key is test_row_0/B:col10/1731708792717/Put/seqid=0 2024-11-15T22:13:14,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742241_1417 (size=12301) 2024-11-15T22:13:14,105 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/535d06ad203b44eb987657a631f39786 2024-11-15T22:13:14,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/7abef5be5fd24eb692f5c6d9af8e92f4 is 50, key is test_row_0/C:col10/1731708792717/Put/seqid=0 2024-11-15T22:13:14,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742242_1418 (size=12301) 2024-11-15T22:13:14,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708854267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:14,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708854267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:14,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:14,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708854267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:14,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708854267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:14,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708854267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T22:13:14,547 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/7abef5be5fd24eb692f5c6d9af8e92f4 2024-11-15T22:13:14,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/3ae6cd7eda7c4405905894576df43bd5 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/3ae6cd7eda7c4405905894576df43bd5 2024-11-15T22:13:14,566 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/3ae6cd7eda7c4405905894576df43bd5, entries=150, sequenceid=280, filesize=30.5 K 2024-11-15T22:13:14,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/535d06ad203b44eb987657a631f39786 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/535d06ad203b44eb987657a631f39786 2024-11-15T22:13:14,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,575 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/535d06ad203b44eb987657a631f39786, entries=150, sequenceid=280, filesize=12.0 K 2024-11-15T22:13:14,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/7abef5be5fd24eb692f5c6d9af8e92f4 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7abef5be5fd24eb692f5c6d9af8e92f4 2024-11-15T22:13:14,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,584 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7abef5be5fd24eb692f5c6d9af8e92f4, entries=150, sequenceid=280, filesize=12.0 K 2024-11-15T22:13:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,585 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 2a98cc30238e089ac8df02a2f0745572 in 967ms, sequenceid=280, compaction requested=false 2024-11-15T22:13:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:14,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:14,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-15T22:13:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-15T22:13:14,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-15T22:13:14,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2840 sec 2024-11-15T22:13:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.2910 sec 2024-11-15T22:13:14,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:14,635 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:14,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T22:13:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156f3684f0593b4796a3df9e0659993991_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708793934/Put/seqid=0 2024-11-15T22:13:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742243_1419 (size=12454) 2024-11-15T22:13:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:14,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708854701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:14,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:14,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708854709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:14,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708854710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:14,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708854712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:14,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:14,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:14,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708854722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:14,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:14,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708854827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:14,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:14,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708854836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:14,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708854836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:14,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708854836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:14,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708854836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:15,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708855046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:15,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:15,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708855057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708855057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708855057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:15,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:15,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708855057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:15,070 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:15,076 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156f3684f0593b4796a3df9e0659993991_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156f3684f0593b4796a3df9e0659993991_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:15,077 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/29993f45efca4bdc85d058b72d305745, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:15,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/29993f45efca4bdc85d058b72d305745 is 175, key is test_row_0/A:col10/1731708793934/Put/seqid=0 2024-11-15T22:13:15,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742244_1420 (size=31255) 2024-11-15T22:13:15,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708855367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:15,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:15,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708855377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:15,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708855377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:15,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:15,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:15,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708855377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:15,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:15,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708855387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:15,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T22:13:15,446 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:15,447 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:13:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-15T22:13:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T22:13:15,449 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:13:15,450 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:13:15,450 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:13:15,490 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/29993f45efca4bdc85d058b72d305745 2024-11-15T22:13:15,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/19825e643213483984642fb2cd67346d is 50, key is test_row_0/B:col10/1731708793934/Put/seqid=0 2024-11-15T22:13:15,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742245_1421 (size=12301) 2024-11-15T22:13:15,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/19825e643213483984642fb2cd67346d 2024-11-15T22:13:15,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/2d3f797c5d5645ad999c6c5597a13210 is 50, key is test_row_0/C:col10/1731708793934/Put/seqid=0 2024-11-15T22:13:15,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742246_1422 (size=12301) 2024-11-15T22:13:15,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/2d3f797c5d5645ad999c6c5597a13210 2024-11-15T22:13:15,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/29993f45efca4bdc85d058b72d305745 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/29993f45efca4bdc85d058b72d305745 2024-11-15T22:13:15,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/29993f45efca4bdc85d058b72d305745, entries=150, sequenceid=296, filesize=30.5 K 2024-11-15T22:13:15,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/19825e643213483984642fb2cd67346d as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/19825e643213483984642fb2cd67346d 2024-11-15T22:13:15,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/19825e643213483984642fb2cd67346d, entries=150, sequenceid=296, filesize=12.0 K 2024-11-15T22:13:15,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/2d3f797c5d5645ad999c6c5597a13210 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/2d3f797c5d5645ad999c6c5597a13210 2024-11-15T22:13:15,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/2d3f797c5d5645ad999c6c5597a13210, entries=150, sequenceid=296, filesize=12.0 K 2024-11-15T22:13:15,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 2a98cc30238e089ac8df02a2f0745572 in 908ms, sequenceid=296, compaction requested=true 2024-11-15T22:13:15,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:15,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:13:15,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:15,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:13:15,543 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:15,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:15,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:13:15,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:15,543 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:15,546 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:15,546 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/A is initiating minor compaction (all files) 2024-11-15T22:13:15,546 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/A in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:15,546 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/08f85d600002405492862852a85e7941, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/3ae6cd7eda7c4405905894576df43bd5, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/29993f45efca4bdc85d058b72d305745] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=92.0 K 2024-11-15T22:13:15,546 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:15,547 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/08f85d600002405492862852a85e7941, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/3ae6cd7eda7c4405905894576df43bd5, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/29993f45efca4bdc85d058b72d305745] 2024-11-15T22:13:15,547 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:15,547 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/B is initiating minor compaction (all files) 2024-11-15T22:13:15,547 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/B in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:15,547 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/28deb18a707249f2bf707490b8d9f38f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/535d06ad203b44eb987657a631f39786, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/19825e643213483984642fb2cd67346d] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=36.5 K 2024-11-15T22:13:15,547 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08f85d600002405492862852a85e7941, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731708791966 2024-11-15T22:13:15,548 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 28deb18a707249f2bf707490b8d9f38f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731708791966 2024-11-15T22:13:15,548 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ae6cd7eda7c4405905894576df43bd5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731708792717 2024-11-15T22:13:15,548 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 535d06ad203b44eb987657a631f39786, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731708792717 2024-11-15T22:13:15,548 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29993f45efca4bdc85d058b72d305745, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731708793933 2024-11-15T22:13:15,549 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 19825e643213483984642fb2cd67346d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731708793933 2024-11-15T22:13:15,555 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:15,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T22:13:15,560 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#B#compaction#348 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:15,560 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/185320ed31724e059c9abb2f527fad91 is 50, key is test_row_0/B:col10/1731708793934/Put/seqid=0 2024-11-15T22:13:15,560 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115f01a1dd861c949c1aa3880ea4d91b7b2_2a98cc30238e089ac8df02a2f0745572 store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:15,562 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115f01a1dd861c949c1aa3880ea4d91b7b2_2a98cc30238e089ac8df02a2f0745572, store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:15,563 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f01a1dd861c949c1aa3880ea4d91b7b2_2a98cc30238e089ac8df02a2f0745572 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:15,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742247_1423 (size=13017) 2024-11-15T22:13:15,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742248_1424 (size=4469) 2024-11-15T22:13:15,589 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/185320ed31724e059c9abb2f527fad91 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/185320ed31724e059c9abb2f527fad91 2024-11-15T22:13:15,597 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/B of 2a98cc30238e089ac8df02a2f0745572 into 185320ed31724e059c9abb2f527fad91(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:15,597 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:15,597 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/B, priority=13, startTime=1731708795543; duration=0sec 2024-11-15T22:13:15,597 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:15,597 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:B 2024-11-15T22:13:15,597 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:15,599 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:15,599 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/C is initiating minor compaction (all files) 2024-11-15T22:13:15,600 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/C in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:15,600 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/3cd3f793587f4b579cba6c04184cf436, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7abef5be5fd24eb692f5c6d9af8e92f4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/2d3f797c5d5645ad999c6c5597a13210] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=36.5 K 2024-11-15T22:13:15,602 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cd3f793587f4b579cba6c04184cf436, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731708791966 2024-11-15T22:13:15,603 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 7abef5be5fd24eb692f5c6d9af8e92f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731708792717 2024-11-15T22:13:15,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T22:13:15,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:15,603 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d3f797c5d5645ad999c6c5597a13210, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731708793933 2024-11-15T22:13:15,603 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T22:13:15,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158a1011ece5c747f2bc7c3898ad5fd77f_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708794712/Put/seqid=0 2024-11-15T22:13:15,628 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#C#compaction#350 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:15,629 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/1d51f5760bfb4194af30d11127be7090 is 50, key is test_row_0/C:col10/1731708793934/Put/seqid=0 2024-11-15T22:13:15,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742249_1425 (size=12454) 2024-11-15T22:13:15,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742250_1426 (size=13017) 2024-11-15T22:13:15,648 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/1d51f5760bfb4194af30d11127be7090 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/1d51f5760bfb4194af30d11127be7090 2024-11-15T22:13:15,660 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/C of 2a98cc30238e089ac8df02a2f0745572 into 1d51f5760bfb4194af30d11127be7090(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:15,660 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:15,660 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/C, priority=13, startTime=1731708795543; duration=0sec 2024-11-15T22:13:15,660 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:15,660 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:C 2024-11-15T22:13:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T22:13:15,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:15,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:15,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708855933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:15,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:15,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:15,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708855935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:15,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708855940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:15,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:15,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708855945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:15,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708855954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:15,983 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#A#compaction#347 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:15,984 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/b25754f09e3e46d296b1b17428c5e603 is 175, key is test_row_0/A:col10/1731708793934/Put/seqid=0 2024-11-15T22:13:15,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742251_1427 (size=31971) 2024-11-15T22:13:16,003 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/b25754f09e3e46d296b1b17428c5e603 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/b25754f09e3e46d296b1b17428c5e603 2024-11-15T22:13:16,010 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/A of 2a98cc30238e089ac8df02a2f0745572 into b25754f09e3e46d296b1b17428c5e603(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:16,010 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:16,010 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/A, priority=13, startTime=1731708795543; duration=0sec 2024-11-15T22:13:16,011 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:16,011 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:A 2024-11-15T22:13:16,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:16,039 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158a1011ece5c747f2bc7c3898ad5fd77f_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158a1011ece5c747f2bc7c3898ad5fd77f_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:16,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/985e10b728b4483ea66f05466b7df341, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:16,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/985e10b728b4483ea66f05466b7df341 is 175, key is test_row_0/A:col10/1731708794712/Put/seqid=0 2024-11-15T22:13:16,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742252_1428 (size=31255) 2024-11-15T22:13:16,060 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/985e10b728b4483ea66f05466b7df341 2024-11-15T22:13:16,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708856058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708856058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/df19f90d95ac40cf801a25c173de3f63 is 50, key is test_row_0/B:col10/1731708794712/Put/seqid=0 2024-11-15T22:13:16,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T22:13:16,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708856066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708856066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708856066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742253_1429 (size=12301) 2024-11-15T22:13:16,085 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/df19f90d95ac40cf801a25c173de3f63 2024-11-15T22:13:16,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/626097a17865402399b7a060eb99e398 is 50, key is test_row_0/C:col10/1731708794712/Put/seqid=0 2024-11-15T22:13:16,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742254_1430 (size=12301) 2024-11-15T22:13:16,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708856276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708856277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708856287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708856289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708856289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,527 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/626097a17865402399b7a060eb99e398 2024-11-15T22:13:16,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/985e10b728b4483ea66f05466b7df341 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/985e10b728b4483ea66f05466b7df341 2024-11-15T22:13:16,539 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/985e10b728b4483ea66f05466b7df341, entries=150, sequenceid=318, filesize=30.5 K 2024-11-15T22:13:16,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/df19f90d95ac40cf801a25c173de3f63 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/df19f90d95ac40cf801a25c173de3f63 2024-11-15T22:13:16,546 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/df19f90d95ac40cf801a25c173de3f63, entries=150, sequenceid=318, filesize=12.0 K 2024-11-15T22:13:16,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/626097a17865402399b7a060eb99e398 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/626097a17865402399b7a060eb99e398 2024-11-15T22:13:16,553 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/626097a17865402399b7a060eb99e398, entries=150, sequenceid=318, filesize=12.0 K 2024-11-15T22:13:16,554 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 2a98cc30238e089ac8df02a2f0745572 in 951ms, sequenceid=318, compaction requested=false 2024-11-15T22:13:16,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:16,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:16,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-15T22:13:16,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-15T22:13:16,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-15T22:13:16,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1060 sec 2024-11-15T22:13:16,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.1110 sec 2024-11-15T22:13:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T22:13:16,586 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:16,587 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:13:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-15T22:13:16,590 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:13:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T22:13:16,590 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:13:16,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:13:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:16,607 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:16,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T22:13:16,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c0c7e880279c4c3683e00f7d5206688e_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708796605/Put/seqid=0 2024-11-15T22:13:16,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742255_1431 (size=12454) 2024-11-15T22:13:16,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708856642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708856643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708856643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708856652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708856653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T22:13:16,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T22:13:16,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:16,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:16,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:16,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:16,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:16,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:16,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708856756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708856767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708856767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708856776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708856776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T22:13:16,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:16,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:16,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:16,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:16,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T22:13:16,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708856967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708856986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708856986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708856986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:16,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708856987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:16,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:16,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:16,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:17,026 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:17,031 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c0c7e880279c4c3683e00f7d5206688e_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c0c7e880279c4c3683e00f7d5206688e_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:17,032 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/f596815540a042a889b22bcad436845c, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:17,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/f596815540a042a889b22bcad436845c is 175, key is test_row_0/A:col10/1731708796605/Put/seqid=0 2024-11-15T22:13:17,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742256_1432 (size=31255) 2024-11-15T22:13:17,045 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=337, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/f596815540a042a889b22bcad436845c 2024-11-15T22:13:17,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T22:13:17,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:17,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:17,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:17,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:17,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:17,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/92de0e12191c4ecc9f36f611cecc18de is 50, key is test_row_0/B:col10/1731708796605/Put/seqid=0 2024-11-15T22:13:17,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742257_1433 (size=12301) 2024-11-15T22:13:17,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T22:13:17,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:17,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:17,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:17,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:17,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:17,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:17,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T22:13:17,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708857286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:17,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:17,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:17,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:17,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708857297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:17,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:17,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:17,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708857297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:17,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:17,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:17,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:17,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:17,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:17,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708857306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:17,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:17,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708857306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:17,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:17,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:17,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:17,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:17,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:17,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:17,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T22:13:17,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:17,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:17,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:17,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:17,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:17,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/92de0e12191c4ecc9f36f611cecc18de 2024-11-15T22:13:17,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/2a203dd4720b4bd6bc3a7d45344eea76 is 50, key is test_row_0/C:col10/1731708796605/Put/seqid=0 2024-11-15T22:13:17,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742258_1434 (size=12301) 2024-11-15T22:13:17,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T22:13:17,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:17,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:17,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:17,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:17,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:17,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:17,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T22:13:17,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:17,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:17,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:17,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:17,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:17,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T22:13:17,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708857797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:17,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:17,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708857817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:17,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:17,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708857817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:17,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:17,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708857826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:17,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:17,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:17,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708857826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:17,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:17,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:17,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:17,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T22:13:17,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:17,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:17,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:17,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:17,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:17,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/2a203dd4720b4bd6bc3a7d45344eea76 2024-11-15T22:13:17,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/f596815540a042a889b22bcad436845c as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/f596815540a042a889b22bcad436845c 2024-11-15T22:13:17,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/f596815540a042a889b22bcad436845c, entries=150, sequenceid=337, filesize=30.5 K 2024-11-15T22:13:17,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/92de0e12191c4ecc9f36f611cecc18de as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/92de0e12191c4ecc9f36f611cecc18de 2024-11-15T22:13:17,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/92de0e12191c4ecc9f36f611cecc18de, entries=150, sequenceid=337, filesize=12.0 K 2024-11-15T22:13:17,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/2a203dd4720b4bd6bc3a7d45344eea76 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/2a203dd4720b4bd6bc3a7d45344eea76 2024-11-15T22:13:17,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/2a203dd4720b4bd6bc3a7d45344eea76, entries=150, sequenceid=337, filesize=12.0 K 2024-11-15T22:13:17,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 2a98cc30238e089ac8df02a2f0745572 in 1337ms, sequenceid=337, compaction requested=true 2024-11-15T22:13:17,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:17,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:13:17,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:17,945 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:17,945 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:17,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:13:17,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:17,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:13:17,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:17,948 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:17,948 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:17,948 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/B is initiating minor compaction (all files) 2024-11-15T22:13:17,948 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/A is initiating minor compaction (all files) 2024-11-15T22:13:17,948 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/A in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:17,948 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/B in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:17,948 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/185320ed31724e059c9abb2f527fad91, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/df19f90d95ac40cf801a25c173de3f63, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/92de0e12191c4ecc9f36f611cecc18de] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=36.7 K 2024-11-15T22:13:17,948 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/b25754f09e3e46d296b1b17428c5e603, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/985e10b728b4483ea66f05466b7df341, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/f596815540a042a889b22bcad436845c] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=92.3 K 2024-11-15T22:13:17,948 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:17,948 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/b25754f09e3e46d296b1b17428c5e603, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/985e10b728b4483ea66f05466b7df341, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/f596815540a042a889b22bcad436845c] 2024-11-15T22:13:17,949 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 185320ed31724e059c9abb2f527fad91, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731708793933 2024-11-15T22:13:17,950 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting b25754f09e3e46d296b1b17428c5e603, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731708793933 2024-11-15T22:13:17,950 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting df19f90d95ac40cf801a25c173de3f63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731708794708 2024-11-15T22:13:17,950 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 92de0e12191c4ecc9f36f611cecc18de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731708795934 2024-11-15T22:13:17,950 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 985e10b728b4483ea66f05466b7df341, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731708794708 2024-11-15T22:13:17,951 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting f596815540a042a889b22bcad436845c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731708795934 2024-11-15T22:13:17,981 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#B#compaction#356 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:17,981 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/bdfbfb90deaf46a1b63546b0ced8a407 is 50, key is test_row_0/B:col10/1731708796605/Put/seqid=0 2024-11-15T22:13:17,993 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:17,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T22:13:17,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:17,997 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T22:13:18,005 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115c62aba070c3c41adb0790f02ab34eaa8_2a98cc30238e089ac8df02a2f0745572 store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:18,006 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115c62aba070c3c41adb0790f02ab34eaa8_2a98cc30238e089ac8df02a2f0745572, store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:18,007 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c62aba070c3c41adb0790f02ab34eaa8_2a98cc30238e089ac8df02a2f0745572 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:18,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111569102237b30a4c7e98560ebe954a353a_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708796651/Put/seqid=0 2024-11-15T22:13:18,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742259_1435 (size=13119) 2024-11-15T22:13:18,065 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/bdfbfb90deaf46a1b63546b0ced8a407 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/bdfbfb90deaf46a1b63546b0ced8a407 2024-11-15T22:13:18,070 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/B of 2a98cc30238e089ac8df02a2f0745572 into bdfbfb90deaf46a1b63546b0ced8a407(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:18,070 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:18,070 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/B, priority=13, startTime=1731708797945; duration=0sec 2024-11-15T22:13:18,070 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:18,070 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:B 2024-11-15T22:13:18,070 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:18,071 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:18,071 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/C is initiating minor compaction (all files) 2024-11-15T22:13:18,071 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/C in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:18,072 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/1d51f5760bfb4194af30d11127be7090, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/626097a17865402399b7a060eb99e398, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/2a203dd4720b4bd6bc3a7d45344eea76] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=36.7 K 2024-11-15T22:13:18,072 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d51f5760bfb4194af30d11127be7090, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731708793933 2024-11-15T22:13:18,072 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 626097a17865402399b7a060eb99e398, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731708794708 2024-11-15T22:13:18,073 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a203dd4720b4bd6bc3a7d45344eea76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731708795934 2024-11-15T22:13:18,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742260_1436 (size=4469) 2024-11-15T22:13:18,084 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#A#compaction#357 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:18,084 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/1de29eaa0e9a4e35a4fbdcd720b784c3 is 175, key is test_row_0/A:col10/1731708796605/Put/seqid=0 2024-11-15T22:13:18,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742261_1437 (size=12454) 2024-11-15T22:13:18,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,127 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#C#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:18,128 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/5bd104ccc42b4736bb8b032ef0003fda is 50, key is test_row_0/C:col10/1731708796605/Put/seqid=0 2024-11-15T22:13:18,137 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111569102237b30a4c7e98560ebe954a353a_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111569102237b30a4c7e98560ebe954a353a_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:18,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742262_1438 (size=32073) 2024-11-15T22:13:18,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/fbdf97c948f74ed4942ef11b4bcdc17d, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:18,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/fbdf97c948f74ed4942ef11b4bcdc17d is 175, key is test_row_0/A:col10/1731708796651/Put/seqid=0 2024-11-15T22:13:18,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742264_1440 (size=31255) 2024-11-15T22:13:18,159 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/fbdf97c948f74ed4942ef11b4bcdc17d 2024-11-15T22:13:18,168 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/1de29eaa0e9a4e35a4fbdcd720b784c3 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/1de29eaa0e9a4e35a4fbdcd720b784c3 2024-11-15T22:13:18,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742263_1439 (size=13119) 2024-11-15T22:13:18,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/a64d105c943e4f6eb947ec6ef0e4101d is 50, key is test_row_0/B:col10/1731708796651/Put/seqid=0 2024-11-15T22:13:18,180 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/5bd104ccc42b4736bb8b032ef0003fda as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/5bd104ccc42b4736bb8b032ef0003fda 2024-11-15T22:13:18,181 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/A of 2a98cc30238e089ac8df02a2f0745572 into 1de29eaa0e9a4e35a4fbdcd720b784c3(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:18,181 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:18,181 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/A, priority=13, startTime=1731708797944; duration=0sec 2024-11-15T22:13:18,181 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:18,181 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:A 2024-11-15T22:13:18,187 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/C of 2a98cc30238e089ac8df02a2f0745572 into 5bd104ccc42b4736bb8b032ef0003fda(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:18,187 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:18,187 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/C, priority=13, startTime=1731708797945; duration=0sec 2024-11-15T22:13:18,187 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:18,187 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:C 2024-11-15T22:13:18,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742265_1441 (size=12301) 2024-11-15T22:13:18,201 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/a64d105c943e4f6eb947ec6ef0e4101d 2024-11-15T22:13:18,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/8df37667958746d3987accfa216f42aa is 50, key is test_row_0/C:col10/1731708796651/Put/seqid=0 2024-11-15T22:13:18,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742266_1442 (size=12301) 2024-11-15T22:13:18,276 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/8df37667958746d3987accfa216f42aa 2024-11-15T22:13:18,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/fbdf97c948f74ed4942ef11b4bcdc17d as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/fbdf97c948f74ed4942ef11b4bcdc17d 2024-11-15T22:13:18,286 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/fbdf97c948f74ed4942ef11b4bcdc17d, entries=150, sequenceid=356, filesize=30.5 K 2024-11-15T22:13:18,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/a64d105c943e4f6eb947ec6ef0e4101d as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/a64d105c943e4f6eb947ec6ef0e4101d 2024-11-15T22:13:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,294 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/a64d105c943e4f6eb947ec6ef0e4101d, entries=150, sequenceid=356, filesize=12.0 K 2024-11-15T22:13:18,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/8df37667958746d3987accfa216f42aa as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/8df37667958746d3987accfa216f42aa 2024-11-15T22:13:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,301 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/8df37667958746d3987accfa216f42aa, entries=150, sequenceid=356, filesize=12.0 K 2024-11-15T22:13:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,302 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for 2a98cc30238e089ac8df02a2f0745572 in 305ms, sequenceid=356, compaction requested=false 2024-11-15T22:13:18,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:18,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:18,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-15T22:13:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-15T22:13:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,307 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-15T22:13:18,307 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7140 sec 2024-11-15T22:13:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.7210 sec 2024-11-15T22:13:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T22:13:18,738 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,738 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:13:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-15T22:13:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,740 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:13:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T22:13:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,741 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:13:18,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:13:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T22:13:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T22:13:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:18,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:18,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-15T22:13:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-15T22:13:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-15T22:13:18,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-11-15T22:13:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 161 msec 2024-11-15T22:13:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,014 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:19,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T22:13:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c2981ffb5aba4a00986e9fe58ed788c7_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708798998/Put/seqid=0 2024-11-15T22:13:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T22:13:19,059 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:19,060 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:13:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-15T22:13:19,062 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:13:19,065 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:13:19,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:13:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T22:13:19,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742268_1444 (size=25158) 2024-11-15T22:13:19,085 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:19,089 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c2981ffb5aba4a00986e9fe58ed788c7_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c2981ffb5aba4a00986e9fe58ed788c7_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:19,090 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/46087ab1b2e548c3906b3f1db61fcabe, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:19,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/46087ab1b2e548c3906b3f1db61fcabe is 175, key is test_row_0/A:col10/1731708798998/Put/seqid=0 2024-11-15T22:13:19,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742267_1443 (size=74795) 2024-11-15T22:13:19,100 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=371, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/46087ab1b2e548c3906b3f1db61fcabe 2024-11-15T22:13:19,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/98a6f5f387a54569b7508a3ca1837e89 is 50, key is test_row_0/B:col10/1731708798998/Put/seqid=0 2024-11-15T22:13:19,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708859103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:19,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:19,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742269_1445 (size=12301) 2024-11-15T22:13:19,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708859128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:19,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708859129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:19,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708859138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:19,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708859138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:19,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T22:13:19,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T22:13:19,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:19,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:19,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:19,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:19,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:19,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708859237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:19,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:19,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708859270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:19,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:19,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708859270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:19,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:19,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:19,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708859270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:19,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708859273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:19,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:19,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:19,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T22:13:19,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:19,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:19,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:19,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:19,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T22:13:19,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:19,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708859457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:19,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708859487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:19,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708859487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:19,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708859487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:19,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:19,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708859488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:19,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:19,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T22:13:19,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:19,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:19,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:19,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:19,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:19,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:19,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/98a6f5f387a54569b7508a3ca1837e89 2024-11-15T22:13:19,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/ba7d17c31a09426488c597dde8461eee is 50, key is test_row_0/C:col10/1731708798998/Put/seqid=0 2024-11-15T22:13:19,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742270_1446 (size=12301) 2024-11-15T22:13:19,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T22:13:19,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:19,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:19,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:19,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:19,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:19,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T22:13:19,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708859777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:19,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:19,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708859796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:19,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:19,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:19,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708859796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:19,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:19,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708859807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:19,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:19,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708859807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:19,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T22:13:19,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:19,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:19,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:19,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:19,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:19,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/ba7d17c31a09426488c597dde8461eee 2024-11-15T22:13:19,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/46087ab1b2e548c3906b3f1db61fcabe as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/46087ab1b2e548c3906b3f1db61fcabe 2024-11-15T22:13:19,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T22:13:19,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/46087ab1b2e548c3906b3f1db61fcabe, entries=400, sequenceid=371, filesize=73.0 K 2024-11-15T22:13:19,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:19,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:19,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:19,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:19,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:20,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/98a6f5f387a54569b7508a3ca1837e89 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/98a6f5f387a54569b7508a3ca1837e89 2024-11-15T22:13:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:20,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/98a6f5f387a54569b7508a3ca1837e89, entries=150, sequenceid=371, filesize=12.0 K 2024-11-15T22:13:20,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/ba7d17c31a09426488c597dde8461eee as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/ba7d17c31a09426488c597dde8461eee 2024-11-15T22:13:20,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/ba7d17c31a09426488c597dde8461eee, entries=150, sequenceid=371, filesize=12.0 K 2024-11-15T22:13:20,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 2a98cc30238e089ac8df02a2f0745572 in 1001ms, sequenceid=371, compaction requested=true 2024-11-15T22:13:20,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:20,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:13:20,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:20,016 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:20,016 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:20,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:13:20,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:20,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:13:20,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:20,017 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 138123 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:20,017 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/A is initiating minor compaction (all files) 2024-11-15T22:13:20,018 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/A in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:20,018 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:20,018 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/1de29eaa0e9a4e35a4fbdcd720b784c3, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/fbdf97c948f74ed4942ef11b4bcdc17d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/46087ab1b2e548c3906b3f1db61fcabe] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=134.9 K 2024-11-15T22:13:20,018 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/B is initiating minor compaction (all files) 2024-11-15T22:13:20,018 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:20,018 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/B in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:20,018 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/1de29eaa0e9a4e35a4fbdcd720b784c3, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/fbdf97c948f74ed4942ef11b4bcdc17d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/46087ab1b2e548c3906b3f1db61fcabe] 2024-11-15T22:13:20,018 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/bdfbfb90deaf46a1b63546b0ced8a407, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/a64d105c943e4f6eb947ec6ef0e4101d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/98a6f5f387a54569b7508a3ca1837e89] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=36.8 K 2024-11-15T22:13:20,018 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1de29eaa0e9a4e35a4fbdcd720b784c3, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731708795934 2024-11-15T22:13:20,019 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting bdfbfb90deaf46a1b63546b0ced8a407, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731708795934 2024-11-15T22:13:20,019 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbdf97c948f74ed4942ef11b4bcdc17d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731708796642 2024-11-15T22:13:20,019 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting a64d105c943e4f6eb947ec6ef0e4101d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731708796642 2024-11-15T22:13:20,019 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46087ab1b2e548c3906b3f1db61fcabe, keycount=400, bloomtype=ROW, size=73.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731708798935 2024-11-15T22:13:20,020 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 98a6f5f387a54569b7508a3ca1837e89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731708798970 2024-11-15T22:13:20,030 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#B#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:20,031 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/3ee950aa05ed47e2a38d1d257f590154 is 50, key is test_row_0/B:col10/1731708798998/Put/seqid=0 2024-11-15T22:13:20,034 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:20,039 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115e4866d8acb7546019ef16717246fa351_2a98cc30238e089ac8df02a2f0745572 store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:20,042 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115e4866d8acb7546019ef16717246fa351_2a98cc30238e089ac8df02a2f0745572, store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:20,043 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e4866d8acb7546019ef16717246fa351_2a98cc30238e089ac8df02a2f0745572 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:20,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742271_1447 (size=13221) 2024-11-15T22:13:20,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742272_1448 (size=4469) 2024-11-15T22:13:20,074 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#A#compaction#366 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:20,074 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/dd181721d5e3461d92532ebd519c5d91 is 175, key is test_row_0/A:col10/1731708798998/Put/seqid=0 2024-11-15T22:13:20,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742273_1449 (size=32175) 2024-11-15T22:13:20,102 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/dd181721d5e3461d92532ebd519c5d91 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/dd181721d5e3461d92532ebd519c5d91 2024-11-15T22:13:20,110 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/A of 2a98cc30238e089ac8df02a2f0745572 into dd181721d5e3461d92532ebd519c5d91(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:20,110 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:20,110 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/A, priority=13, startTime=1731708800016; duration=0sec 2024-11-15T22:13:20,110 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:20,110 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:A 2024-11-15T22:13:20,110 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:20,111 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:20,112 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/C is initiating minor compaction (all files) 2024-11-15T22:13:20,112 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/C in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:20,112 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/5bd104ccc42b4736bb8b032ef0003fda, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/8df37667958746d3987accfa216f42aa, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/ba7d17c31a09426488c597dde8461eee] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=36.8 K 2024-11-15T22:13:20,112 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bd104ccc42b4736bb8b032ef0003fda, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731708795934 2024-11-15T22:13:20,114 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8df37667958746d3987accfa216f42aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731708796642 2024-11-15T22:13:20,115 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba7d17c31a09426488c597dde8461eee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731708798970 2024-11-15T22:13:20,134 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#C#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:20,135 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/2935e0a5b9944c8e8559f4f16499493e is 50, key is test_row_0/C:col10/1731708798998/Put/seqid=0 2024-11-15T22:13:20,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-15T22:13:20,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:20,154 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T22:13:20,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742274_1450 (size=13221) 2024-11-15T22:13:20,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c19eee636d0c477f99c4f5f2e81af491_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708799123/Put/seqid=0 2024-11-15T22:13:20,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742275_1451 (size=12454) 2024-11-15T22:13:20,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:20,183 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c19eee636d0c477f99c4f5f2e81af491_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c19eee636d0c477f99c4f5f2e81af491_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:20,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/45ff6ca9121847b19dfff4195fb73e00, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:20,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/45ff6ca9121847b19dfff4195fb73e00 is 175, key is test_row_0/A:col10/1731708799123/Put/seqid=0 2024-11-15T22:13:20,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742276_1452 (size=31255) 2024-11-15T22:13:20,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T22:13:20,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:20,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708860329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:20,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:20,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708860330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:20,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:20,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:20,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708860336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:20,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:20,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708860337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:20,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708860337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:20,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708860446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:20,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708860446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:20,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708860446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:20,467 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/3ee950aa05ed47e2a38d1d257f590154 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/3ee950aa05ed47e2a38d1d257f590154 2024-11-15T22:13:20,473 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/B of 2a98cc30238e089ac8df02a2f0745572 into 3ee950aa05ed47e2a38d1d257f590154(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:20,473 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:20,473 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/B, priority=13, startTime=1731708800016; duration=0sec 2024-11-15T22:13:20,473 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:20,473 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:B 2024-11-15T22:13:20,568 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/2935e0a5b9944c8e8559f4f16499493e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/2935e0a5b9944c8e8559f4f16499493e 2024-11-15T22:13:20,581 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/C of 2a98cc30238e089ac8df02a2f0745572 into 2935e0a5b9944c8e8559f4f16499493e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:20,581 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:20,581 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/C, priority=13, startTime=1731708800016; duration=0sec 2024-11-15T22:13:20,583 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:20,583 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:C 2024-11-15T22:13:20,598 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=398, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/45ff6ca9121847b19dfff4195fb73e00 2024-11-15T22:13:20,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/c924338cea5c47d3be0b42b808bc4b0f is 50, key is test_row_0/B:col10/1731708799123/Put/seqid=0 2024-11-15T22:13:20,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742277_1453 (size=12301) 2024-11-15T22:13:20,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:20,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708860666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:20,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708860666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:20,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:20,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708860667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:20,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:20,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:20,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708860977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:20,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:20,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708860987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:21,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:21,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708860989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:21,055 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/c924338cea5c47d3be0b42b808bc4b0f 2024-11-15T22:13:21,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/e3bb2f93f8014367928b724711d29f56 is 50, key is test_row_0/C:col10/1731708799123/Put/seqid=0 2024-11-15T22:13:21,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742278_1454 (size=12301) 2024-11-15T22:13:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T22:13:21,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:21,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708861357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:21,362 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_2' in 2a98cc30238e089ac8df02a2f0745572 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:13:21,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:21,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:21,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:21,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:21,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708861366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:21,372 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_0' in 2a98cc30238e089ac8df02a2f0745572 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:13:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:21,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:21,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708861496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:21,506 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/e3bb2f93f8014367928b724711d29f56 2024-11-15T22:13:21,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708861507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:21,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:21,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:21,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:21,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708861507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:21,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:21,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:21,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:21,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/45ff6ca9121847b19dfff4195fb73e00 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/45ff6ca9121847b19dfff4195fb73e00 2024-11-15T22:13:21,523 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/45ff6ca9121847b19dfff4195fb73e00, entries=150, sequenceid=398, filesize=30.5 K 2024-11-15T22:13:21,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/c924338cea5c47d3be0b42b808bc4b0f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/c924338cea5c47d3be0b42b808bc4b0f 2024-11-15T22:13:21,534 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/c924338cea5c47d3be0b42b808bc4b0f, entries=150, sequenceid=398, filesize=12.0 K 2024-11-15T22:13:21,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/e3bb2f93f8014367928b724711d29f56 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e3bb2f93f8014367928b724711d29f56 2024-11-15T22:13:21,542 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e3bb2f93f8014367928b724711d29f56, entries=150, sequenceid=398, filesize=12.0 K 2024-11-15T22:13:21,551 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 2a98cc30238e089ac8df02a2f0745572 in 1397ms, sequenceid=398, compaction requested=false 2024-11-15T22:13:21,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:21,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:21,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-15T22:13:21,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-15T22:13:21,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-15T22:13:21,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4880 sec 2024-11-15T22:13:21,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.4980 sec 2024-11-15T22:13:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:22,511 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:22,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T22:13:22,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a1c9501e8e604b21b228efdd3ae933e1_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708802509/Put/seqid=0 2024-11-15T22:13:22,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742279_1455 (size=12454) 2024-11-15T22:13:22,546 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:22,550 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a1c9501e8e604b21b228efdd3ae933e1_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a1c9501e8e604b21b228efdd3ae933e1_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:22,551 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/106463831c0f4f50a27503ca941a7d55, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:22,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/106463831c0f4f50a27503ca941a7d55 is 175, key is test_row_0/A:col10/1731708802509/Put/seqid=0 2024-11-15T22:13:22,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742280_1456 (size=31255) 2024-11-15T22:13:22,585 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=411, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/106463831c0f4f50a27503ca941a7d55 2024-11-15T22:13:22,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/93da060cc7db48fe9f8243e43dc9490f is 50, key is test_row_0/B:col10/1731708802509/Put/seqid=0 2024-11-15T22:13:22,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742281_1457 (size=12301) 2024-11-15T22:13:22,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/93da060cc7db48fe9f8243e43dc9490f 2024-11-15T22:13:22,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/ec287ac6937948b480880697291e7949 is 50, key is test_row_0/C:col10/1731708802509/Put/seqid=0 2024-11-15T22:13:22,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708862626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:22,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708862638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:22,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708862640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:22,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742282_1458 (size=12301) 2024-11-15T22:13:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:22,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/ec287ac6937948b480880697291e7949 2024-11-15T22:13:22,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/106463831c0f4f50a27503ca941a7d55 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/106463831c0f4f50a27503ca941a7d55 2024-11-15T22:13:22,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/106463831c0f4f50a27503ca941a7d55, entries=150, sequenceid=411, filesize=30.5 K 2024-11-15T22:13:22,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/93da060cc7db48fe9f8243e43dc9490f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/93da060cc7db48fe9f8243e43dc9490f 2024-11-15T22:13:22,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/93da060cc7db48fe9f8243e43dc9490f, entries=150, sequenceid=411, filesize=12.0 K 2024-11-15T22:13:22,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/ec287ac6937948b480880697291e7949 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/ec287ac6937948b480880697291e7949 2024-11-15T22:13:22,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/ec287ac6937948b480880697291e7949, entries=150, sequenceid=411, filesize=12.0 K 2024-11-15T22:13:22,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2a98cc30238e089ac8df02a2f0745572 in 199ms, sequenceid=411, compaction requested=true 2024-11-15T22:13:22,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:22,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:13:22,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:22,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:13:22,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:13:22,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2a98cc30238e089ac8df02a2f0745572:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:13:22,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T22:13:22,711 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:22,711 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:22,715 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:22,715 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/C is initiating minor compaction (all files) 2024-11-15T22:13:22,715 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/C in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:22,716 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/2935e0a5b9944c8e8559f4f16499493e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e3bb2f93f8014367928b724711d29f56, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/ec287ac6937948b480880697291e7949] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=36.9 K 2024-11-15T22:13:22,716 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:22,716 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/A is initiating minor compaction (all files) 2024-11-15T22:13:22,716 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/A in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:22,716 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/dd181721d5e3461d92532ebd519c5d91, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/45ff6ca9121847b19dfff4195fb73e00, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/106463831c0f4f50a27503ca941a7d55] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=92.5 K 2024-11-15T22:13:22,716 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:22,716 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/dd181721d5e3461d92532ebd519c5d91, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/45ff6ca9121847b19dfff4195fb73e00, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/106463831c0f4f50a27503ca941a7d55] 2024-11-15T22:13:22,717 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd181721d5e3461d92532ebd519c5d91, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731708798970 2024-11-15T22:13:22,717 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 2935e0a5b9944c8e8559f4f16499493e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731708798970 2024-11-15T22:13:22,719 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45ff6ca9121847b19dfff4195fb73e00, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731708799089 2024-11-15T22:13:22,719 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting e3bb2f93f8014367928b724711d29f56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731708799089 2024-11-15T22:13:22,720 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting ec287ac6937948b480880697291e7949, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731708800314 2024-11-15T22:13:22,720 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 106463831c0f4f50a27503ca941a7d55, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731708800314 2024-11-15T22:13:22,742 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:22,747 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#C#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:22,748 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/54bfef304aa84b00b2d2c764d3d6b174 is 50, key is test_row_0/C:col10/1731708802509/Put/seqid=0 2024-11-15T22:13:22,758 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411150e4e906406544ed7bae2bf7e3cd23528_2a98cc30238e089ac8df02a2f0745572 store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:22,760 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411150e4e906406544ed7bae2bf7e3cd23528_2a98cc30238e089ac8df02a2f0745572, store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:22,760 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150e4e906406544ed7bae2bf7e3cd23528_2a98cc30238e089ac8df02a2f0745572 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:22,762 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:22,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T22:13:22,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742283_1459 (size=13323) 2024-11-15T22:13:22,804 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/54bfef304aa84b00b2d2c764d3d6b174 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/54bfef304aa84b00b2d2c764d3d6b174 2024-11-15T22:13:22,816 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/C of 2a98cc30238e089ac8df02a2f0745572 into 54bfef304aa84b00b2d2c764d3d6b174(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:22,816 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:22,816 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/C, priority=13, startTime=1731708802711; duration=0sec 2024-11-15T22:13:22,816 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:22,816 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:C 2024-11-15T22:13:22,816 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:22,817 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:22,817 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 2a98cc30238e089ac8df02a2f0745572/B is initiating minor compaction (all files) 2024-11-15T22:13:22,817 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2a98cc30238e089ac8df02a2f0745572/B in TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:22,817 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/3ee950aa05ed47e2a38d1d257f590154, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/c924338cea5c47d3be0b42b808bc4b0f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/93da060cc7db48fe9f8243e43dc9490f] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp, totalSize=36.9 K 2024-11-15T22:13:22,818 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ee950aa05ed47e2a38d1d257f590154, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731708798970 2024-11-15T22:13:22,818 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting c924338cea5c47d3be0b42b808bc4b0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731708799089 2024-11-15T22:13:22,819 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 93da060cc7db48fe9f8243e43dc9490f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731708800314 2024-11-15T22:13:22,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742284_1460 (size=4469) 2024-11-15T22:13:22,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708862804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:22,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:22,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:22,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:22,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154aec32b17f6b447eb1eeb110f4e6954d_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708802637/Put/seqid=0 2024-11-15T22:13:22,831 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#B#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:22,832 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/43f40ef821af4dacb2da7db8d276aa05 is 50, key is test_row_0/B:col10/1731708802509/Put/seqid=0 2024-11-15T22:13:22,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708862811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:22,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708862814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:22,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:22,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742285_1461 (size=13323) 2024-11-15T22:13:22,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742286_1462 (size=14994) 2024-11-15T22:13:22,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708862927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:22,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:22,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:22,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708862946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:22,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708862949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:22,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:22,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:22,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:22,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:23,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708863146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:23,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:23,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:23,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:23,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708863167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:23,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:23,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:23,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:23,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708863177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:23,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:23,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:23,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:23,222 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2a98cc30238e089ac8df02a2f0745572#A#compaction#374 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:23,223 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/4235c60ecb844913958cf67f427048e5 is 175, key is test_row_0/A:col10/1731708802509/Put/seqid=0 2024-11-15T22:13:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T22:13:23,227 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:23,228 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:13:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-15T22:13:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T22:13:23,230 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:13:23,231 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:13:23,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:13:23,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742287_1463 (size=32277) 2024-11-15T22:13:23,293 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/43f40ef821af4dacb2da7db8d276aa05 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/43f40ef821af4dacb2da7db8d276aa05 2024-11-15T22:13:23,296 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:23,301 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/B of 2a98cc30238e089ac8df02a2f0745572 into 43f40ef821af4dacb2da7db8d276aa05(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:23,301 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:23,301 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/B, priority=13, startTime=1731708802711; duration=0sec 2024-11-15T22:13:23,301 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:23,301 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:B 2024-11-15T22:13:23,302 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154aec32b17f6b447eb1eeb110f4e6954d_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154aec32b17f6b447eb1eeb110f4e6954d_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:23,303 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/4a9322c6b614421180a32e744dd6f7ae, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:23,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/4a9322c6b614421180a32e744dd6f7ae is 175, key is test_row_0/A:col10/1731708802637/Put/seqid=0 2024-11-15T22:13:23,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742288_1464 (size=39949) 2024-11-15T22:13:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T22:13:23,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39172 deadline: 1731708863377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:23,380 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_2' in 2a98cc30238e089ac8df02a2f0745572 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:13:23,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:23,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:23,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:23,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T22:13:23,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:23,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:23,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:23,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:23,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:23,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39130 deadline: 1731708863392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:23,406 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_0' in 2a98cc30238e089ac8df02a2f0745572 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4268 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:13:23,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:23,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:23,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:23,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708863462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:23,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708863487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:23,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708863499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:23,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T22:13:23,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:23,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:23,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:23,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:23,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T22:13:23,646 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/4235c60ecb844913958cf67f427048e5 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/4235c60ecb844913958cf67f427048e5 2024-11-15T22:13:23,657 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2a98cc30238e089ac8df02a2f0745572/A of 2a98cc30238e089ac8df02a2f0745572 into 4235c60ecb844913958cf67f427048e5(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:23,657 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:23,657 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., storeName=2a98cc30238e089ac8df02a2f0745572/A, priority=13, startTime=1731708802710; duration=0sec 2024-11-15T22:13:23,657 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:23,657 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2a98cc30238e089ac8df02a2f0745572:A 2024-11-15T22:13:23,697 INFO [Thread-1561 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1561. 2024-11-15T22:13:23,697 DEBUG [Thread-1561 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:13:23,697 DEBUG [Thread-1561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:23,697 DEBUG [Thread-1561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:23,697 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:23,704 INFO [Thread-1559 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1559. 2024-11-15T22:13:23,704 DEBUG [Thread-1559 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:13:23,704 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:23,704 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:23,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T22:13:23,704 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:23,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:23,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:23,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:23,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:23,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:23,706 INFO [Thread-1563 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1563. 2024-11-15T22:13:23,706 DEBUG [Thread-1563 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:13:23,706 DEBUG [Thread-1563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:23,706 DEBUG [Thread-1563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:23,707 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:23,710 INFO [Thread-1555 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1555. 2024-11-15T22:13:23,710 DEBUG [Thread-1555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:13:23,710 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:23,710 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:23,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:23,713 INFO [Thread-1557 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1557. 2024-11-15T22:13:23,713 DEBUG [Thread-1557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:13:23,713 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:23,713 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:23,713 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:23,716 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=437, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/4a9322c6b614421180a32e744dd6f7ae 2024-11-15T22:13:23,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/8861aaefaf0b4586b3bb46b0680803fb is 50, key is test_row_0/B:col10/1731708802637/Put/seqid=0 2024-11-15T22:13:23,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742289_1465 (size=12301) 2024-11-15T22:13:23,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/8861aaefaf0b4586b3bb46b0680803fb 2024-11-15T22:13:23,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/4caab6b9e6e74c82b5a9a70b0e927704 is 50, key is test_row_0/C:col10/1731708802637/Put/seqid=0 2024-11-15T22:13:23,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742290_1466 (size=12301) 2024-11-15T22:13:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T22:13:23,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T22:13:23,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:23,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:23,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:23,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:23,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:23,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39132 deadline: 1731708863986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:24,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:24,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39148 deadline: 1731708864006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39174 deadline: 1731708864006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:24,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:24,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:24,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:24,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:24,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:24,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:13:24,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T22:13:24,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:24,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:24,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:24,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:24,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:24,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/4caab6b9e6e74c82b5a9a70b0e927704 2024-11-15T22:13:24,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/4a9322c6b614421180a32e744dd6f7ae as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/4a9322c6b614421180a32e744dd6f7ae 2024-11-15T22:13:24,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/4a9322c6b614421180a32e744dd6f7ae, entries=200, sequenceid=437, filesize=39.0 K 2024-11-15T22:13:24,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T22:13:24,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:24,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:24,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:24,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/8861aaefaf0b4586b3bb46b0680803fb as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/8861aaefaf0b4586b3bb46b0680803fb 2024-11-15T22:13:24,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:24,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:24,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:24,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/8861aaefaf0b4586b3bb46b0680803fb, entries=150, sequenceid=437, filesize=12.0 K 2024-11-15T22:13:24,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/4caab6b9e6e74c82b5a9a70b0e927704 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/4caab6b9e6e74c82b5a9a70b0e927704 2024-11-15T22:13:24,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/4caab6b9e6e74c82b5a9a70b0e927704, entries=150, sequenceid=437, filesize=12.0 K 2024-11-15T22:13:24,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 2a98cc30238e089ac8df02a2f0745572 in 1423ms, sequenceid=437, compaction requested=false 2024-11-15T22:13:24,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:24,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T22:13:24,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:24,322 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T22:13:24,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d34dc59a2d2843f7bcf784fe4be85004_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708802773/Put/seqid=0 2024-11-15T22:13:24,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742291_1467 (size=12454) 2024-11-15T22:13:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T22:13:24,574 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T22:13:24,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:24,735 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d34dc59a2d2843f7bcf784fe4be85004_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d34dc59a2d2843f7bcf784fe4be85004_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:24,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/4fcd9530490f4a699eab74f383b8ed87, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:24,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/4fcd9530490f4a699eab74f383b8ed87 is 175, key is test_row_0/A:col10/1731708802773/Put/seqid=0 2024-11-15T22:13:24,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742292_1468 (size=31255) 2024-11-15T22:13:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:24,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. as already flushing 2024-11-15T22:13:24,998 INFO [Thread-1544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1544. 2024-11-15T22:13:24,998 DEBUG [Thread-1544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:13:24,998 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:24,998 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:24,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:25,018 INFO [Thread-1546 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1546. 2024-11-15T22:13:25,018 INFO [Thread-1552 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1552. 2024-11-15T22:13:25,018 DEBUG [Thread-1546 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:13:25,018 DEBUG [Thread-1552 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:13:25,018 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:25,018 DEBUG [Thread-1552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:25,018 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:25,019 DEBUG [Thread-1552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:25,019 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:25,019 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:25,142 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=451, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/4fcd9530490f4a699eab74f383b8ed87 2024-11-15T22:13:25,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/fe28c005f3644474ad490d19b13ded29 is 50, key is test_row_0/B:col10/1731708802773/Put/seqid=0 2024-11-15T22:13:25,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742293_1469 (size=12301) 2024-11-15T22:13:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T22:13:25,551 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/fe28c005f3644474ad490d19b13ded29 2024-11-15T22:13:25,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/73f0f1ab265d458bb0acd42351bea2cf is 50, key is test_row_0/C:col10/1731708802773/Put/seqid=0 2024-11-15T22:13:25,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742294_1470 (size=12301) 2024-11-15T22:13:25,962 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/73f0f1ab265d458bb0acd42351bea2cf 2024-11-15T22:13:25,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/4fcd9530490f4a699eab74f383b8ed87 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/4fcd9530490f4a699eab74f383b8ed87 2024-11-15T22:13:25,969 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/4fcd9530490f4a699eab74f383b8ed87, entries=150, sequenceid=451, filesize=30.5 K 2024-11-15T22:13:25,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/fe28c005f3644474ad490d19b13ded29 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/fe28c005f3644474ad490d19b13ded29 2024-11-15T22:13:25,972 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/fe28c005f3644474ad490d19b13ded29, entries=150, sequenceid=451, filesize=12.0 K 2024-11-15T22:13:25,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/73f0f1ab265d458bb0acd42351bea2cf as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/73f0f1ab265d458bb0acd42351bea2cf 2024-11-15T22:13:25,975 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/73f0f1ab265d458bb0acd42351bea2cf, entries=150, sequenceid=451, filesize=12.0 K 2024-11-15T22:13:25,976 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=20.13 KB/20610 for 2a98cc30238e089ac8df02a2f0745572 in 1654ms, sequenceid=451, compaction requested=true 2024-11-15T22:13:25,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 2a98cc30238e089ac8df02a2f0745572: 2024-11-15T22:13:25,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:25,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-15T22:13:25,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-15T22:13:25,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-15T22:13:25,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7460 sec 2024-11-15T22:13:25,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.7510 sec 2024-11-15T22:13:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T22:13:27,387 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:27,418 INFO [Thread-1550 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1550. 2024-11-15T22:13:27,419 DEBUG [Thread-1550 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:13:27,419 DEBUG [Thread-1550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:27,419 DEBUG [Thread-1550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:27,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:27,428 INFO [Thread-1548 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1548. 2024-11-15T22:13:27,428 DEBUG [Thread-1548 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:13:27,428 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:27,428 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:27,428 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:27,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T22:13:27,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-15T22:13:27,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-15T22:13:27,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-15T22:13:27,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-15T22:13:27,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-15T22:13:27,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T22:13:27,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T22:13:27,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 968 2024-11-15T22:13:27,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2896 rows 2024-11-15T22:13:27,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 969 2024-11-15T22:13:27,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2907 rows 2024-11-15T22:13:27,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 959 2024-11-15T22:13:27,429 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2877 rows 2024-11-15T22:13:27,429 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 970 2024-11-15T22:13:27,429 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2910 rows 2024-11-15T22:13:27,429 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 970 2024-11-15T22:13:27,429 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2910 rows 2024-11-15T22:13:27,429 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T22:13:27,429 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:13:27,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:27,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:27,429 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:27,430 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T22:13:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:27,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T22:13:27,433 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708807433"}]},"ts":"1731708807433"} 2024-11-15T22:13:27,434 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T22:13:27,434 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T22:13:27,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T22:13:27,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a98cc30238e089ac8df02a2f0745572, UNASSIGN}] 2024-11-15T22:13:27,437 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a98cc30238e089ac8df02a2f0745572, UNASSIGN 2024-11-15T22:13:27,438 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=2a98cc30238e089ac8df02a2f0745572, regionState=CLOSING, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:27,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a98cc30238e089ac8df02a2f0745572, UNASSIGN because future has completed 2024-11-15T22:13:27,439 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T22:13:27,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516}] 2024-11-15T22:13:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T22:13:27,590 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(122): Close 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:27,590 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T22:13:27,590 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1722): Closing 2a98cc30238e089ac8df02a2f0745572, disabling compactions & flushes 2024-11-15T22:13:27,590 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:27,590 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:27,590 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. after waiting 0 ms 2024-11-15T22:13:27,591 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:27,591 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(2902): Flushing 2a98cc30238e089ac8df02a2f0745572 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T22:13:27,596 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111531731adefb78447d9c9219f95caf2b9f_2a98cc30238e089ac8df02a2f0745572 is 50, key is test_row_0/A:col10/1731708807426/Put/seqid=0 2024-11-15T22:13:27,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742295_1471 (size=12454) 2024-11-15T22:13:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T22:13:27,999 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:28,003 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111531731adefb78447d9c9219f95caf2b9f_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111531731adefb78447d9c9219f95caf2b9f_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:28,003 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/6026bbb6f3fc459f9158e362f2a2b0ae, store: [table=TestAcidGuarantees family=A region=2a98cc30238e089ac8df02a2f0745572] 2024-11-15T22:13:28,004 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/6026bbb6f3fc459f9158e362f2a2b0ae is 175, key is test_row_0/A:col10/1731708807426/Put/seqid=0 2024-11-15T22:13:28,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742296_1472 (size=31255) 2024-11-15T22:13:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T22:13:28,408 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=459, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/6026bbb6f3fc459f9158e362f2a2b0ae 2024-11-15T22:13:28,416 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/237f1fe5c5f04ff58973584549e44c2b is 50, key is test_row_0/B:col10/1731708807426/Put/seqid=0 2024-11-15T22:13:28,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742297_1473 (size=12301) 2024-11-15T22:13:28,431 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/237f1fe5c5f04ff58973584549e44c2b 2024-11-15T22:13:28,439 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/d5273c9e8de642d5b77d1775a8087bd3 is 50, key is test_row_0/C:col10/1731708807426/Put/seqid=0 2024-11-15T22:13:28,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742298_1474 (size=12301) 2024-11-15T22:13:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T22:13:28,663 WARN [regionserver/ae3311ea5ccb:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-15T22:13:28,663 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/8784d22482454ff3a4c0bee3a0c864e8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/ca656a2279f14de6a1e9a56c2ce96944, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/ddee843a9f4f4b62ac15939df7b51450, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/e6f1f0478abd43408ee23fb350b94efd, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/f97e3e894ec54212b112e725710f8f3a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/daadc4a0f270438e939f3c9b0eec71a7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/97034a49ee044ebf9918835cbded17b0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/c449399b7db04829b11f4387138cce07, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/7a437a4d5e1b45c995c99086475e439f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/8876c9713b0b43f0919fe5501ecd364f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/6743620a97ad406bb6e352b3bbfe9539, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/1215a3de8bb04d47b7253366ac13c142, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/46c1be5e9f934f53ae40879068d40b47, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/90c3ee31a12b4b2e8a73ff641732ad8d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/918e1c647bca427ca1b99c1542131797, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/ca28651442a74208933d4a233ac33ede, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/5b34f975dd344157b986074777afee54, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/b9047a12afc04d1ebf904bfe7eda5267, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/08f85d600002405492862852a85e7941, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/3ae6cd7eda7c4405905894576df43bd5, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/b25754f09e3e46d296b1b17428c5e603, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/29993f45efca4bdc85d058b72d305745, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/985e10b728b4483ea66f05466b7df341, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/1de29eaa0e9a4e35a4fbdcd720b784c3, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/f596815540a042a889b22bcad436845c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/fbdf97c948f74ed4942ef11b4bcdc17d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/46087ab1b2e548c3906b3f1db61fcabe, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/dd181721d5e3461d92532ebd519c5d91, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/45ff6ca9121847b19dfff4195fb73e00, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/106463831c0f4f50a27503ca941a7d55] to archive 2024-11-15T22:13:28,664 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:13:28,666 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/8784d22482454ff3a4c0bee3a0c864e8 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/8784d22482454ff3a4c0bee3a0c864e8 2024-11-15T22:13:28,668 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/ca656a2279f14de6a1e9a56c2ce96944 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/ca656a2279f14de6a1e9a56c2ce96944 2024-11-15T22:13:28,669 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/ddee843a9f4f4b62ac15939df7b51450 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/ddee843a9f4f4b62ac15939df7b51450 2024-11-15T22:13:28,670 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/e6f1f0478abd43408ee23fb350b94efd to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/e6f1f0478abd43408ee23fb350b94efd 2024-11-15T22:13:28,671 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/f97e3e894ec54212b112e725710f8f3a to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/f97e3e894ec54212b112e725710f8f3a 2024-11-15T22:13:28,673 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/daadc4a0f270438e939f3c9b0eec71a7 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/daadc4a0f270438e939f3c9b0eec71a7 2024-11-15T22:13:28,674 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/97034a49ee044ebf9918835cbded17b0 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/97034a49ee044ebf9918835cbded17b0 2024-11-15T22:13:28,675 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/c449399b7db04829b11f4387138cce07 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/c449399b7db04829b11f4387138cce07 2024-11-15T22:13:28,676 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/7a437a4d5e1b45c995c99086475e439f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/7a437a4d5e1b45c995c99086475e439f 2024-11-15T22:13:28,678 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/8876c9713b0b43f0919fe5501ecd364f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/8876c9713b0b43f0919fe5501ecd364f 2024-11-15T22:13:28,680 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/6743620a97ad406bb6e352b3bbfe9539 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/6743620a97ad406bb6e352b3bbfe9539 2024-11-15T22:13:28,681 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/1215a3de8bb04d47b7253366ac13c142 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/1215a3de8bb04d47b7253366ac13c142 2024-11-15T22:13:28,686 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/46c1be5e9f934f53ae40879068d40b47 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/46c1be5e9f934f53ae40879068d40b47 2024-11-15T22:13:28,688 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/90c3ee31a12b4b2e8a73ff641732ad8d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/90c3ee31a12b4b2e8a73ff641732ad8d 2024-11-15T22:13:28,689 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/918e1c647bca427ca1b99c1542131797 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/918e1c647bca427ca1b99c1542131797 2024-11-15T22:13:28,690 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/ca28651442a74208933d4a233ac33ede to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/ca28651442a74208933d4a233ac33ede 2024-11-15T22:13:28,692 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/5b34f975dd344157b986074777afee54 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/5b34f975dd344157b986074777afee54 2024-11-15T22:13:28,693 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/b9047a12afc04d1ebf904bfe7eda5267 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/b9047a12afc04d1ebf904bfe7eda5267 2024-11-15T22:13:28,694 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/08f85d600002405492862852a85e7941 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/08f85d600002405492862852a85e7941 2024-11-15T22:13:28,695 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/3ae6cd7eda7c4405905894576df43bd5 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/3ae6cd7eda7c4405905894576df43bd5 2024-11-15T22:13:28,697 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/b25754f09e3e46d296b1b17428c5e603 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/b25754f09e3e46d296b1b17428c5e603 2024-11-15T22:13:28,698 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/29993f45efca4bdc85d058b72d305745 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/29993f45efca4bdc85d058b72d305745 2024-11-15T22:13:28,699 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/985e10b728b4483ea66f05466b7df341 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/985e10b728b4483ea66f05466b7df341 2024-11-15T22:13:28,701 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/1de29eaa0e9a4e35a4fbdcd720b784c3 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/1de29eaa0e9a4e35a4fbdcd720b784c3 2024-11-15T22:13:28,702 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/f596815540a042a889b22bcad436845c to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/f596815540a042a889b22bcad436845c 2024-11-15T22:13:28,708 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/fbdf97c948f74ed4942ef11b4bcdc17d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/fbdf97c948f74ed4942ef11b4bcdc17d 2024-11-15T22:13:28,710 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/46087ab1b2e548c3906b3f1db61fcabe to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/46087ab1b2e548c3906b3f1db61fcabe 2024-11-15T22:13:28,712 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/dd181721d5e3461d92532ebd519c5d91 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/dd181721d5e3461d92532ebd519c5d91 2024-11-15T22:13:28,713 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/45ff6ca9121847b19dfff4195fb73e00 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/45ff6ca9121847b19dfff4195fb73e00 2024-11-15T22:13:28,714 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/106463831c0f4f50a27503ca941a7d55 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/106463831c0f4f50a27503ca941a7d55 2024-11-15T22:13:28,723 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/ecbef8b4599449b29bc1f8512d0e3252, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/11d920103fde40dda4f245e8282deddc, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/fd225ccb4a724ee1bc317501584932ff, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/839d7cb9bf6c49b0a8fd75257a3b619c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/ba9d5a6ec2aa450b8b69c42b0d77702c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/b95fb6097065424ba808150b6be0d294, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/0602d26d71d2423c9f17015e1c90af94, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/f726a790cf1b438b9b9df990bca620ab, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/37d901232e69424fa136c009f09ac9b3, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/127fc688710347dcbf901ca6258b5862, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/b077008ac27846788282c5deb7b52544, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/77ced7c49f5e4d43bc9c1c0a6764699f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/5e69b604c26440d4bfaa8684039ca6fb, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/3daac38e8b1a48b497422faa351893b2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/9b2037fe74904ece884b1ecee4cbd4de, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/15cdcc8cf3f84ae59dcda1aaff127be1, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/55957fbe4f534852803282614c5ee85c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/28deb18a707249f2bf707490b8d9f38f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/74c0ff4f13ae4720ac16aeece9d42307, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/535d06ad203b44eb987657a631f39786, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/185320ed31724e059c9abb2f527fad91, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/19825e643213483984642fb2cd67346d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/df19f90d95ac40cf801a25c173de3f63, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/bdfbfb90deaf46a1b63546b0ced8a407, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/92de0e12191c4ecc9f36f611cecc18de, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/a64d105c943e4f6eb947ec6ef0e4101d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/3ee950aa05ed47e2a38d1d257f590154, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/98a6f5f387a54569b7508a3ca1837e89, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/c924338cea5c47d3be0b42b808bc4b0f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/93da060cc7db48fe9f8243e43dc9490f] to archive 2024-11-15T22:13:28,725 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:13:28,729 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/ecbef8b4599449b29bc1f8512d0e3252 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/ecbef8b4599449b29bc1f8512d0e3252 2024-11-15T22:13:28,730 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/11d920103fde40dda4f245e8282deddc to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/11d920103fde40dda4f245e8282deddc 2024-11-15T22:13:28,732 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/fd225ccb4a724ee1bc317501584932ff to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/fd225ccb4a724ee1bc317501584932ff 2024-11-15T22:13:28,733 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/839d7cb9bf6c49b0a8fd75257a3b619c to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/839d7cb9bf6c49b0a8fd75257a3b619c 2024-11-15T22:13:28,734 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/ba9d5a6ec2aa450b8b69c42b0d77702c to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/ba9d5a6ec2aa450b8b69c42b0d77702c 2024-11-15T22:13:28,737 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/b95fb6097065424ba808150b6be0d294 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/b95fb6097065424ba808150b6be0d294 2024-11-15T22:13:28,738 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/0602d26d71d2423c9f17015e1c90af94 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/0602d26d71d2423c9f17015e1c90af94 2024-11-15T22:13:28,739 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/f726a790cf1b438b9b9df990bca620ab to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/f726a790cf1b438b9b9df990bca620ab 2024-11-15T22:13:28,740 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/37d901232e69424fa136c009f09ac9b3 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/37d901232e69424fa136c009f09ac9b3 2024-11-15T22:13:28,741 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/127fc688710347dcbf901ca6258b5862 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/127fc688710347dcbf901ca6258b5862 2024-11-15T22:13:28,742 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/b077008ac27846788282c5deb7b52544 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/b077008ac27846788282c5deb7b52544 2024-11-15T22:13:28,743 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/77ced7c49f5e4d43bc9c1c0a6764699f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/77ced7c49f5e4d43bc9c1c0a6764699f 2024-11-15T22:13:28,744 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/5e69b604c26440d4bfaa8684039ca6fb to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/5e69b604c26440d4bfaa8684039ca6fb 2024-11-15T22:13:28,745 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/3daac38e8b1a48b497422faa351893b2 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/3daac38e8b1a48b497422faa351893b2 2024-11-15T22:13:28,746 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/9b2037fe74904ece884b1ecee4cbd4de to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/9b2037fe74904ece884b1ecee4cbd4de 2024-11-15T22:13:28,748 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/15cdcc8cf3f84ae59dcda1aaff127be1 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/15cdcc8cf3f84ae59dcda1aaff127be1 2024-11-15T22:13:28,749 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/55957fbe4f534852803282614c5ee85c to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/55957fbe4f534852803282614c5ee85c 2024-11-15T22:13:28,750 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/28deb18a707249f2bf707490b8d9f38f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/28deb18a707249f2bf707490b8d9f38f 2024-11-15T22:13:28,751 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/74c0ff4f13ae4720ac16aeece9d42307 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/74c0ff4f13ae4720ac16aeece9d42307 2024-11-15T22:13:28,752 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/535d06ad203b44eb987657a631f39786 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/535d06ad203b44eb987657a631f39786 2024-11-15T22:13:28,753 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/185320ed31724e059c9abb2f527fad91 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/185320ed31724e059c9abb2f527fad91 2024-11-15T22:13:28,754 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/19825e643213483984642fb2cd67346d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/19825e643213483984642fb2cd67346d 2024-11-15T22:13:28,755 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/df19f90d95ac40cf801a25c173de3f63 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/df19f90d95ac40cf801a25c173de3f63 2024-11-15T22:13:28,757 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/bdfbfb90deaf46a1b63546b0ced8a407 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/bdfbfb90deaf46a1b63546b0ced8a407 2024-11-15T22:13:28,758 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/92de0e12191c4ecc9f36f611cecc18de to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/92de0e12191c4ecc9f36f611cecc18de 2024-11-15T22:13:28,759 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/a64d105c943e4f6eb947ec6ef0e4101d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/a64d105c943e4f6eb947ec6ef0e4101d 2024-11-15T22:13:28,760 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/3ee950aa05ed47e2a38d1d257f590154 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/3ee950aa05ed47e2a38d1d257f590154 2024-11-15T22:13:28,761 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/98a6f5f387a54569b7508a3ca1837e89 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/98a6f5f387a54569b7508a3ca1837e89 2024-11-15T22:13:28,763 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/c924338cea5c47d3be0b42b808bc4b0f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/c924338cea5c47d3be0b42b808bc4b0f 2024-11-15T22:13:28,764 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/93da060cc7db48fe9f8243e43dc9490f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/93da060cc7db48fe9f8243e43dc9490f 2024-11-15T22:13:28,770 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e1de6822faa24552a7a9385c709c6584, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/6cb1c455821c4a2489802d9fe9f79565, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/1ac38a6212e74467ba6696c8127bc211, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/a59b243fc6064d0e8974db73d7ab7019, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/0e564e128ed04264819162fdc68823c0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/6121a0b8c97443e4a10bd7a2397cf401, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e8a2d4ebe990444989851c71fc37d035, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/9a6d6da6ea154adf90575eaab5d22290, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7f8167b87f6d449588fd1c03f93d9275, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/303af36a57b644d6a9e1cc01525dd9e3, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/3d20ca780c8045d086ee849e33f05682, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/d8ef443458bc4a60aef77cdccde15a3e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/d270dd3bf07f43388b3d15fe1fe21234, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/14f042b2194f4ab5964e0e704a683bcc, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7f777af3c64d440898944807430f1197, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e4f1d19b388349c1ba267aadddf291a6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/d9c39ad10cc347b083c9cdd758f403c4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/3cd3f793587f4b579cba6c04184cf436, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7ffa78386e2b43e0866770473643f49b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7abef5be5fd24eb692f5c6d9af8e92f4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/1d51f5760bfb4194af30d11127be7090, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/2d3f797c5d5645ad999c6c5597a13210, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/626097a17865402399b7a060eb99e398, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/5bd104ccc42b4736bb8b032ef0003fda, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/2a203dd4720b4bd6bc3a7d45344eea76, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/8df37667958746d3987accfa216f42aa, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/2935e0a5b9944c8e8559f4f16499493e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/ba7d17c31a09426488c597dde8461eee, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e3bb2f93f8014367928b724711d29f56, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/ec287ac6937948b480880697291e7949] to archive 2024-11-15T22:13:28,776 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:13:28,789 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e1de6822faa24552a7a9385c709c6584 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e1de6822faa24552a7a9385c709c6584 2024-11-15T22:13:28,793 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/6cb1c455821c4a2489802d9fe9f79565 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/6cb1c455821c4a2489802d9fe9f79565 2024-11-15T22:13:28,795 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/1ac38a6212e74467ba6696c8127bc211 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/1ac38a6212e74467ba6696c8127bc211 2024-11-15T22:13:28,796 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/a59b243fc6064d0e8974db73d7ab7019 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/a59b243fc6064d0e8974db73d7ab7019 2024-11-15T22:13:28,797 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/0e564e128ed04264819162fdc68823c0 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/0e564e128ed04264819162fdc68823c0 2024-11-15T22:13:28,798 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/6121a0b8c97443e4a10bd7a2397cf401 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/6121a0b8c97443e4a10bd7a2397cf401 2024-11-15T22:13:28,799 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e8a2d4ebe990444989851c71fc37d035 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e8a2d4ebe990444989851c71fc37d035 2024-11-15T22:13:28,800 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/9a6d6da6ea154adf90575eaab5d22290 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/9a6d6da6ea154adf90575eaab5d22290 2024-11-15T22:13:28,802 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7f8167b87f6d449588fd1c03f93d9275 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7f8167b87f6d449588fd1c03f93d9275 2024-11-15T22:13:28,804 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/303af36a57b644d6a9e1cc01525dd9e3 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/303af36a57b644d6a9e1cc01525dd9e3 2024-11-15T22:13:28,805 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/3d20ca780c8045d086ee849e33f05682 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/3d20ca780c8045d086ee849e33f05682 2024-11-15T22:13:28,806 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/d8ef443458bc4a60aef77cdccde15a3e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/d8ef443458bc4a60aef77cdccde15a3e 2024-11-15T22:13:28,809 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/d270dd3bf07f43388b3d15fe1fe21234 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/d270dd3bf07f43388b3d15fe1fe21234 2024-11-15T22:13:28,811 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/14f042b2194f4ab5964e0e704a683bcc to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/14f042b2194f4ab5964e0e704a683bcc 2024-11-15T22:13:28,812 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7f777af3c64d440898944807430f1197 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7f777af3c64d440898944807430f1197 2024-11-15T22:13:28,813 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e4f1d19b388349c1ba267aadddf291a6 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e4f1d19b388349c1ba267aadddf291a6 2024-11-15T22:13:28,815 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/d9c39ad10cc347b083c9cdd758f403c4 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/d9c39ad10cc347b083c9cdd758f403c4 2024-11-15T22:13:28,816 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/3cd3f793587f4b579cba6c04184cf436 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/3cd3f793587f4b579cba6c04184cf436 2024-11-15T22:13:28,818 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7ffa78386e2b43e0866770473643f49b to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7ffa78386e2b43e0866770473643f49b 2024-11-15T22:13:28,819 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7abef5be5fd24eb692f5c6d9af8e92f4 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/7abef5be5fd24eb692f5c6d9af8e92f4 2024-11-15T22:13:28,820 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/1d51f5760bfb4194af30d11127be7090 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/1d51f5760bfb4194af30d11127be7090 2024-11-15T22:13:28,821 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/2d3f797c5d5645ad999c6c5597a13210 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/2d3f797c5d5645ad999c6c5597a13210 2024-11-15T22:13:28,823 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/626097a17865402399b7a060eb99e398 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/626097a17865402399b7a060eb99e398 2024-11-15T22:13:28,824 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/5bd104ccc42b4736bb8b032ef0003fda to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/5bd104ccc42b4736bb8b032ef0003fda 2024-11-15T22:13:28,825 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/2a203dd4720b4bd6bc3a7d45344eea76 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/2a203dd4720b4bd6bc3a7d45344eea76 2024-11-15T22:13:28,826 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/8df37667958746d3987accfa216f42aa to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/8df37667958746d3987accfa216f42aa 2024-11-15T22:13:28,827 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/2935e0a5b9944c8e8559f4f16499493e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/2935e0a5b9944c8e8559f4f16499493e 2024-11-15T22:13:28,828 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/ba7d17c31a09426488c597dde8461eee to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/ba7d17c31a09426488c597dde8461eee 2024-11-15T22:13:28,829 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e3bb2f93f8014367928b724711d29f56 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/e3bb2f93f8014367928b724711d29f56 2024-11-15T22:13:28,830 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/ec287ac6937948b480880697291e7949 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/ec287ac6937948b480880697291e7949 2024-11-15T22:13:28,847 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/d5273c9e8de642d5b77d1775a8087bd3 2024-11-15T22:13:28,852 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/A/6026bbb6f3fc459f9158e362f2a2b0ae as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/6026bbb6f3fc459f9158e362f2a2b0ae 2024-11-15T22:13:28,856 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/6026bbb6f3fc459f9158e362f2a2b0ae, entries=150, sequenceid=459, filesize=30.5 K 2024-11-15T22:13:28,856 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/B/237f1fe5c5f04ff58973584549e44c2b as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/237f1fe5c5f04ff58973584549e44c2b 2024-11-15T22:13:28,861 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/237f1fe5c5f04ff58973584549e44c2b, entries=150, sequenceid=459, filesize=12.0 K 2024-11-15T22:13:28,862 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/.tmp/C/d5273c9e8de642d5b77d1775a8087bd3 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/d5273c9e8de642d5b77d1775a8087bd3 2024-11-15T22:13:28,866 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/d5273c9e8de642d5b77d1775a8087bd3, entries=150, sequenceid=459, filesize=12.0 K 2024-11-15T22:13:28,867 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 2a98cc30238e089ac8df02a2f0745572 in 1276ms, sequenceid=459, compaction requested=true 2024-11-15T22:13:28,897 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/recovered.edits/462.seqid, newMaxSeqId=462, maxSeqId=4 2024-11-15T22:13:28,898 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. 2024-11-15T22:13:28,898 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1676): Region close journal for 2a98cc30238e089ac8df02a2f0745572: Waiting for close lock at 1731708807590Running coprocessor pre-close hooks at 1731708807590Disabling compacts and flushes for region at 1731708807590Disabling writes for close at 1731708807591 (+1 ms)Obtaining lock to block concurrent updates at 1731708807591Preparing flush snapshotting stores in 2a98cc30238e089ac8df02a2f0745572 at 1731708807591Finished memstore snapshotting TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731708807591Flushing stores of TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572. at 1731708807591Flushing 2a98cc30238e089ac8df02a2f0745572/A: creating writer at 1731708807591Flushing mob file 2a98cc30238e089ac8df02a2f0745572/A: appending metadata at 1731708807596 (+5 ms)Flushing mob file 2a98cc30238e089ac8df02a2f0745572/A: closing flushed file at 1731708807596Flushing 2a98cc30238e089ac8df02a2f0745572/A: appending metadata at 1731708808003 (+407 ms)Flushing 2a98cc30238e089ac8df02a2f0745572/A: closing flushed file at 1731708808004 (+1 ms)Flushing 2a98cc30238e089ac8df02a2f0745572/B: creating writer at 1731708808412 (+408 ms)Flushing 2a98cc30238e089ac8df02a2f0745572/B: appending metadata at 1731708808415 (+3 ms)Flushing 2a98cc30238e089ac8df02a2f0745572/B: closing flushed file at 1731708808415Flushing 2a98cc30238e089ac8df02a2f0745572/C: creating writer at 1731708808435 (+20 ms)Flushing 2a98cc30238e089ac8df02a2f0745572/C: appending metadata at 1731708808438 (+3 ms)Flushing 2a98cc30238e089ac8df02a2f0745572/C: closing flushed file at 1731708808438Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31a8c875: reopening flushed file at 1731708808851 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12c157ec: reopening flushed file at 1731708808856 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70062536: reopening flushed file at 1731708808861 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 2a98cc30238e089ac8df02a2f0745572 in 1276ms, sequenceid=459, compaction requested=true at 1731708808867 (+6 ms)Writing region close event to WAL at 1731708808883 (+16 ms)Running coprocessor post-close hooks at 1731708808898 (+15 ms)Closed at 1731708808898 2024-11-15T22:13:28,901 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(157): Closed 2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:28,902 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=2a98cc30238e089ac8df02a2f0745572, regionState=CLOSED 2024-11-15T22:13:28,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 because future has completed 2024-11-15T22:13:28,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-15T22:13:28,907 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseRegionProcedure 2a98cc30238e089ac8df02a2f0745572, server=ae3311ea5ccb,37917,1731708686516 in 1.4650 sec 2024-11-15T22:13:28,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-15T22:13:28,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2a98cc30238e089ac8df02a2f0745572, UNASSIGN in 1.4710 sec 2024-11-15T22:13:28,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-15T22:13:28,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4790 sec 2024-11-15T22:13:28,917 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708808917"}]},"ts":"1731708808917"} 2024-11-15T22:13:28,919 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T22:13:28,919 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T22:13:28,921 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4900 sec 2024-11-15T22:13:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T22:13:29,577 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:29,577 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T22:13:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:29,579 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T22:13:29,580 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=123, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:29,581 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,583 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A, FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B, FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C, FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/recovered.edits] 2024-11-15T22:13:29,585 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/4235c60ecb844913958cf67f427048e5 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/4235c60ecb844913958cf67f427048e5 2024-11-15T22:13:29,586 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/4a9322c6b614421180a32e744dd6f7ae to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/4a9322c6b614421180a32e744dd6f7ae 2024-11-15T22:13:29,587 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/4fcd9530490f4a699eab74f383b8ed87 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/4fcd9530490f4a699eab74f383b8ed87 2024-11-15T22:13:29,588 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/6026bbb6f3fc459f9158e362f2a2b0ae to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/A/6026bbb6f3fc459f9158e362f2a2b0ae 2024-11-15T22:13:29,589 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/237f1fe5c5f04ff58973584549e44c2b to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/237f1fe5c5f04ff58973584549e44c2b 2024-11-15T22:13:29,590 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/43f40ef821af4dacb2da7db8d276aa05 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/43f40ef821af4dacb2da7db8d276aa05 2024-11-15T22:13:29,591 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/8861aaefaf0b4586b3bb46b0680803fb to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/8861aaefaf0b4586b3bb46b0680803fb 2024-11-15T22:13:29,592 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/fe28c005f3644474ad490d19b13ded29 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/B/fe28c005f3644474ad490d19b13ded29 2024-11-15T22:13:29,594 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/4caab6b9e6e74c82b5a9a70b0e927704 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/4caab6b9e6e74c82b5a9a70b0e927704 2024-11-15T22:13:29,595 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/54bfef304aa84b00b2d2c764d3d6b174 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/54bfef304aa84b00b2d2c764d3d6b174 2024-11-15T22:13:29,596 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/73f0f1ab265d458bb0acd42351bea2cf to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/73f0f1ab265d458bb0acd42351bea2cf 2024-11-15T22:13:29,596 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/d5273c9e8de642d5b77d1775a8087bd3 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/C/d5273c9e8de642d5b77d1775a8087bd3 2024-11-15T22:13:29,598 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/recovered.edits/462.seqid to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572/recovered.edits/462.seqid 2024-11-15T22:13:29,599 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,599 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T22:13:29,599 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T22:13:29,600 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T22:13:29,602 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115241b7f1a53f14b4d9f765ac789484d62_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115241b7f1a53f14b4d9f765ac789484d62_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,603 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111527a4c9569abd4e32a12c1258f3bd6660_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111527a4c9569abd4e32a12c1258f3bd6660_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,604 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111531731adefb78447d9c9219f95caf2b9f_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111531731adefb78447d9c9219f95caf2b9f_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,605 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111531ea8eafe15d405d8d95dfef25464d29_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111531ea8eafe15d405d8d95dfef25464d29_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,606 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115343f49de939345e4937fa7f30f8119c0_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115343f49de939345e4937fa7f30f8119c0_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,607 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111537c408ac7fbd456eb24207a59f08428c_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111537c408ac7fbd456eb24207a59f08428c_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,608 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115486a8d32b3b84ed99e2cf6da60e53d4c_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115486a8d32b3b84ed99e2cf6da60e53d4c_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,609 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154aec32b17f6b447eb1eeb110f4e6954d_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154aec32b17f6b447eb1eeb110f4e6954d_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,610 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111569102237b30a4c7e98560ebe954a353a_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111569102237b30a4c7e98560ebe954a353a_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,611 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156f3684f0593b4796a3df9e0659993991_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156f3684f0593b4796a3df9e0659993991_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,612 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158a1011ece5c747f2bc7c3898ad5fd77f_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158a1011ece5c747f2bc7c3898ad5fd77f_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,613 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158c0392fdeabd4feea56458331728a331_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158c0392fdeabd4feea56458331728a331_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,614 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a1c9501e8e604b21b228efdd3ae933e1_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a1c9501e8e604b21b228efdd3ae933e1_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,615 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115adbd8c75031b4b3aa3f00dde1b4d65d0_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115adbd8c75031b4b3aa3f00dde1b4d65d0_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,616 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c0c7e880279c4c3683e00f7d5206688e_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c0c7e880279c4c3683e00f7d5206688e_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,617 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c19eee636d0c477f99c4f5f2e81af491_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c19eee636d0c477f99c4f5f2e81af491_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,618 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c2981ffb5aba4a00986e9fe58ed788c7_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c2981ffb5aba4a00986e9fe58ed788c7_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,619 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c5ec27426c4a4141bcaa6042c67710aa_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c5ec27426c4a4141bcaa6042c67710aa_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,620 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c7a0725a67b14fe59a26f2b657e54de9_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c7a0725a67b14fe59a26f2b657e54de9_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,621 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cbcbdd000f1d48d8bf3f2eeb8dd4ca0d_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cbcbdd000f1d48d8bf3f2eeb8dd4ca0d_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,621 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cfc02a9f9a8a4e688dc5f94619c58218_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cfc02a9f9a8a4e688dc5f94619c58218_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,622 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d34dc59a2d2843f7bcf784fe4be85004_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d34dc59a2d2843f7bcf784fe4be85004_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,623 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d6ba9c92d5e34723b9763a3a0039339d_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d6ba9c92d5e34723b9763a3a0039339d_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,624 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115de61289ef81946c8922fb0b872701907_2a98cc30238e089ac8df02a2f0745572 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115de61289ef81946c8922fb0b872701907_2a98cc30238e089ac8df02a2f0745572 2024-11-15T22:13:29,624 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T22:13:29,626 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=123, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:29,628 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T22:13:29,630 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T22:13:29,631 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=123, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:29,631 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T22:13:29,631 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731708809631"}]},"ts":"9223372036854775807"} 2024-11-15T22:13:29,633 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T22:13:29,633 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2a98cc30238e089ac8df02a2f0745572, NAME => 'TestAcidGuarantees,,1731708780541.2a98cc30238e089ac8df02a2f0745572.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T22:13:29,633 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T22:13:29,633 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731708809633"}]},"ts":"9223372036854775807"} 2024-11-15T22:13:29,634 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T22:13:29,635 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=123, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:29,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 58 msec 2024-11-15T22:13:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T22:13:29,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T22:13:29,687 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:29,698 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 238) - Thread LEAK? -, OpenFileDescriptor=459 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=710 (was 632) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2643 (was 3440) 2024-11-15T22:13:29,710 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=710, ProcessCount=11, AvailableMemoryMB=2642 2024-11-15T22:13:29,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T22:13:29,712 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:13:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:29,714 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T22:13:29,714 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:29,714 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 124 2024-11-15T22:13:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-15T22:13:29,715 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T22:13:29,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742299_1475 (size=959) 2024-11-15T22:13:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-15T22:13:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-15T22:13:30,125 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31 2024-11-15T22:13:30,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742300_1476 (size=53) 2024-11-15T22:13:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-15T22:13:30,530 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:13:30,530 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d0014c64eb4fbb0d0723a1402347f3ea, disabling compactions & flushes 2024-11-15T22:13:30,530 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:30,530 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:30,530 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. after waiting 0 ms 2024-11-15T22:13:30,530 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:30,530 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:30,530 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d0014c64eb4fbb0d0723a1402347f3ea: Waiting for close lock at 1731708810530Disabling compacts and flushes for region at 1731708810530Disabling writes for close at 1731708810530Writing region close event to WAL at 1731708810530Closed at 1731708810530 2024-11-15T22:13:30,531 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T22:13:30,531 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731708810531"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708810531"}]},"ts":"1731708810531"} 2024-11-15T22:13:30,533 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T22:13:30,533 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T22:13:30,533 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708810533"}]},"ts":"1731708810533"} 2024-11-15T22:13:30,534 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T22:13:30,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0014c64eb4fbb0d0723a1402347f3ea, ASSIGN}] 2024-11-15T22:13:30,535 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0014c64eb4fbb0d0723a1402347f3ea, ASSIGN 2024-11-15T22:13:30,536 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0014c64eb4fbb0d0723a1402347f3ea, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,37917,1731708686516; forceNewPlan=false, retain=false 2024-11-15T22:13:30,687 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=d0014c64eb4fbb0d0723a1402347f3ea, regionState=OPENING, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:30,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0014c64eb4fbb0d0723a1402347f3ea, ASSIGN because future has completed 2024-11-15T22:13:30,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516}] 2024-11-15T22:13:30,843 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:30,843 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7752): Opening region: {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:13:30,843 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:30,843 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:13:30,843 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7794): checking encryption for d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:30,843 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7797): checking classloading for d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:30,845 INFO [StoreOpener-d0014c64eb4fbb0d0723a1402347f3ea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:30,846 INFO [StoreOpener-d0014c64eb4fbb0d0723a1402347f3ea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0014c64eb4fbb0d0723a1402347f3ea columnFamilyName A 2024-11-15T22:13:30,846 DEBUG [StoreOpener-d0014c64eb4fbb0d0723a1402347f3ea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:30,846 INFO [StoreOpener-d0014c64eb4fbb0d0723a1402347f3ea-1 {}] regionserver.HStore(327): Store=d0014c64eb4fbb0d0723a1402347f3ea/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:13:30,846 INFO [StoreOpener-d0014c64eb4fbb0d0723a1402347f3ea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:30,847 INFO [StoreOpener-d0014c64eb4fbb0d0723a1402347f3ea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0014c64eb4fbb0d0723a1402347f3ea columnFamilyName B 2024-11-15T22:13:30,847 DEBUG [StoreOpener-d0014c64eb4fbb0d0723a1402347f3ea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:30,848 INFO [StoreOpener-d0014c64eb4fbb0d0723a1402347f3ea-1 {}] regionserver.HStore(327): Store=d0014c64eb4fbb0d0723a1402347f3ea/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:13:30,848 INFO [StoreOpener-d0014c64eb4fbb0d0723a1402347f3ea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:30,849 INFO [StoreOpener-d0014c64eb4fbb0d0723a1402347f3ea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0014c64eb4fbb0d0723a1402347f3ea columnFamilyName C 2024-11-15T22:13:30,849 DEBUG [StoreOpener-d0014c64eb4fbb0d0723a1402347f3ea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:30,849 INFO [StoreOpener-d0014c64eb4fbb0d0723a1402347f3ea-1 {}] regionserver.HStore(327): Store=d0014c64eb4fbb0d0723a1402347f3ea/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:13:30,849 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1038): replaying wal for d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:30,850 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:30,850 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:30,851 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1048): stopping wal replay for d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:30,851 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1060): Cleaning up temporary data for d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:30,851 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:13:30,852 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): writing seq id for d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:30,853 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:13:30,854 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1114): Opened d0014c64eb4fbb0d0723a1402347f3ea; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74402381, jitterRate=0.10868187248706818}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:13:30,854 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:30,854 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1006): Region open journal for d0014c64eb4fbb0d0723a1402347f3ea: Running coprocessor pre-open hook at 1731708810844Writing region info on filesystem at 1731708810844Initializing all the Stores at 1731708810844Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708810844Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708810844Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708810844Cleaning up temporary data from old regions at 1731708810851 (+7 ms)Running coprocessor post-open hooks at 1731708810854 (+3 ms)Region opened successfully at 1731708810854 2024-11-15T22:13:30,855 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., pid=126, masterSystemTime=1731708810840 2024-11-15T22:13:30,857 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:30,857 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-15T22:13:30,857 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=d0014c64eb4fbb0d0723a1402347f3ea, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:30,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 because future has completed 2024-11-15T22:13:30,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-15T22:13:30,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; OpenRegionProcedure d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 in 171 msec 2024-11-15T22:13:30,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-15T22:13:30,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0014c64eb4fbb0d0723a1402347f3ea, ASSIGN in 326 msec 2024-11-15T22:13:30,863 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T22:13:30,863 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708810863"}]},"ts":"1731708810863"} 2024-11-15T22:13:30,864 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T22:13:30,864 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T22:13:30,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-11-15T22:13:31,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-15T22:13:31,867 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:31,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bfd8f06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,868 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:31,868 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:31,869 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@475e9016, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:31,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:31,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:31,871 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:31,871 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a6707f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,871 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:31,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:31,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:31,873 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:31,874 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T22:13:31,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is ae3311ea5ccb,37303,1731708685620 2024-11-15T22:13:31,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@54fd66d4 2024-11-15T22:13:31,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T22:13:31,875 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T22:13:31,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d395438, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,876 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:31,876 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:31,877 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17f0a5bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:31,878 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:31,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33a44ed1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2d6985, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,879 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:31,879 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:31,880 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5be0af84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:31,881 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:31,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c1951d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@490627bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,882 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:31,882 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:31,883 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:31,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:31,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:31,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@288c927a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:31,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:31,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:31,884 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:31,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a17dba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,885 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e16e9b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,885 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:31,885 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:31,886 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:31,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:31,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:31,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a465b77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:31,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:31,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:31,887 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b038ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a6735, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,888 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:31,888 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:31,888 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:31,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:31,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:31,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5329b309, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:31,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:31,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:31,889 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:31,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72ff25e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f3c4fbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,890 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:31,891 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:31,891 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:31,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:31,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:31,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30fa6fc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:31,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:31,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:31,892 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:31,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5af4a49a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@732b24ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,893 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:31,893 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:31,894 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db19027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:31,895 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30471acc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eaa8538, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:31,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:31,896 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:31,896 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:31,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:31,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ac7c067, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:31,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:31,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:31,897 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:31,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749d2d30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67669ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,899 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:31,899 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:31,899 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:31,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:31,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:31,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59a4f84e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:31,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:31,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:31,901 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:31,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65e62e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69e4731f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:13:31,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:13:31,902 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:13:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:13:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:13:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a72f27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:13:31,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:13:31,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:31,903 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:13:31,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@345f5767, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:13:31,905 DEBUG [Thread-2138 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:31,905 DEBUG [Thread-2129 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:31,905 DEBUG [Thread-2125 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:31,905 DEBUG [Thread-2140 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:31,905 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:13:31,905 DEBUG [Thread-2144 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:31,906 DEBUG [Thread-2133 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:31,906 DEBUG [Thread-2131 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:31,906 DEBUG [Thread-2127 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:31,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-15T22:13:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:31,907 DEBUG [Thread-2136 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:31,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:31,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:31,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:31,908 DEBUG [Thread-2142 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:13:31,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T22:13:31,908 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:13:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:31,909 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:31,909 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:31,909 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:31,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:31,909 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:31,910 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:31,910 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:13:31,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:13:31,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:13:31,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:13:31,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:13:31,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:13:31,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:13:31,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:31,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:13:31,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:13:31,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:13:31,913 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:13:31,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:13:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:13:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:13:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2] 2024-11-15T22:13:31,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:31,922 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:31,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0014c64eb4fbb0d0723a1402347f3ea 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T22:13:31,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/4b0bbbfe01f944d8815f761c4f4b6f37 is 50, key is test_row_0/A:col10/1731708811921/Put/seqid=0 2024-11-15T22:13:31,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742301_1477 (size=14341) 2024-11-15T22:13:31,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:31,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708871962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:31,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:31,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:31,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:31,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:31,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708871966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:31,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:31,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:31,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708871975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:31,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:31,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:31,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:31,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:31,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:31,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:31,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708871975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:31,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:31,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:31,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:31,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:31,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708871976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:31,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:31,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:31,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:32,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T22:13:32,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T22:13:32,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:32,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:32,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:32,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:32,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:32,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:32,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:32,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:32,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708872087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:32,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708872087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:32,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:32,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:32,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708872087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:32,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708872087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:32,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:32,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:32,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:32,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:32,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:32,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708872096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:32,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:32,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T22:13:32,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:32,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:32,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:32,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:32,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:32,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T22:13:32,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:32,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708872306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:32,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:32,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:32,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708872306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:32,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:32,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:32,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708872306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:32,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708872306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:32,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:32,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708872316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:32,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:32,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/4b0bbbfe01f944d8815f761c4f4b6f37 2024-11-15T22:13:32,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T22:13:32,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:32,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:32,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:32,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:32,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:32,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:32,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/94cd9872ef5f429c8279e0fd6308cb8f is 50, key is test_row_0/B:col10/1731708811921/Put/seqid=0 2024-11-15T22:13:32,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742302_1478 (size=12001) 2024-11-15T22:13:32,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T22:13:32,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:32,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:32,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:32,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:32,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:32,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:32,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T22:13:32,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:32,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708872616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:32,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708872616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:32,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708872616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:32,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708872626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:32,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:32,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708872637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:32,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:32,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:32,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T22:13:32,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:32,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:32,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:32,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:32,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:32,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:32,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/94cd9872ef5f429c8279e0fd6308cb8f 2024-11-15T22:13:32,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/a535f1294611440cba285fa234080780 is 50, key is test_row_0/C:col10/1731708811921/Put/seqid=0 2024-11-15T22:13:32,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742303_1479 (size=12001) 2024-11-15T22:13:32,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T22:13:32,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:32,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:32,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:32,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:32,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:32,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T22:13:32,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:32,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:32,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:32,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:32,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:32,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T22:13:33,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:33,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708873126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:33,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:33,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:33,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:33,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708873126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:33,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:33,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:33,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708873126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:33,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:33,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:33,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:33,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:33,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T22:13:33,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:33,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:33,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:33,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:33,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:33,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:33,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:33,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708873146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:33,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:33,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:33,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:33,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708873156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:33,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/a535f1294611440cba285fa234080780 2024-11-15T22:13:33,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/4b0bbbfe01f944d8815f761c4f4b6f37 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/4b0bbbfe01f944d8815f761c4f4b6f37 2024-11-15T22:13:33,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/4b0bbbfe01f944d8815f761c4f4b6f37, entries=200, sequenceid=13, filesize=14.0 K 2024-11-15T22:13:33,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/94cd9872ef5f429c8279e0fd6308cb8f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/94cd9872ef5f429c8279e0fd6308cb8f 2024-11-15T22:13:33,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/94cd9872ef5f429c8279e0fd6308cb8f, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T22:13:33,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/a535f1294611440cba285fa234080780 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/a535f1294611440cba285fa234080780 2024-11-15T22:13:33,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/a535f1294611440cba285fa234080780, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T22:13:33,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for d0014c64eb4fbb0d0723a1402347f3ea in 1315ms, sequenceid=13, compaction requested=false 2024-11-15T22:13:33,237 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T22:13:33,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:33,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T22:13:33,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:33,291 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing d0014c64eb4fbb0d0723a1402347f3ea 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T22:13:33,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/85148d3424e04bd69960b414511ecbc7 is 50, key is test_row_0/A:col10/1731708811974/Put/seqid=0 2024-11-15T22:13:33,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742304_1480 (size=12001) 2024-11-15T22:13:33,701 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/85148d3424e04bd69960b414511ecbc7 2024-11-15T22:13:33,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/ffbb4bd2cadd433b932da698f1081af1 is 50, key is test_row_0/B:col10/1731708811974/Put/seqid=0 2024-11-15T22:13:33,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742305_1481 (size=12001) 2024-11-15T22:13:34,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T22:13:34,117 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/ffbb4bd2cadd433b932da698f1081af1 2024-11-15T22:13:34,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/5cafb610329b4aa0b400033da1a45982 is 50, key is test_row_0/C:col10/1731708811974/Put/seqid=0 2024-11-15T22:13:34,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742306_1482 (size=12001) 2024-11-15T22:13:34,153 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/5cafb610329b4aa0b400033da1a45982 2024-11-15T22:13:34,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:34,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:34,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/85148d3424e04bd69960b414511ecbc7 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/85148d3424e04bd69960b414511ecbc7 2024-11-15T22:13:34,164 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/85148d3424e04bd69960b414511ecbc7, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T22:13:34,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/ffbb4bd2cadd433b932da698f1081af1 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ffbb4bd2cadd433b932da698f1081af1 2024-11-15T22:13:34,167 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ffbb4bd2cadd433b932da698f1081af1, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T22:13:34,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/5cafb610329b4aa0b400033da1a45982 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/5cafb610329b4aa0b400033da1a45982 2024-11-15T22:13:34,172 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/5cafb610329b4aa0b400033da1a45982, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T22:13:34,173 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for d0014c64eb4fbb0d0723a1402347f3ea in 882ms, sequenceid=38, compaction requested=false 2024-11-15T22:13:34,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:34,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:34,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-15T22:13:34,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-15T22:13:34,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-15T22:13:34,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2650 sec 2024-11-15T22:13:34,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.2720 sec 2024-11-15T22:13:34,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:34,179 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:34,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0014c64eb4fbb0d0723a1402347f3ea 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T22:13:34,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/a0d3df1f399b4281a6c7f63aa53ba7a1 is 50, key is test_row_0/A:col10/1731708814161/Put/seqid=0 2024-11-15T22:13:34,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742307_1483 (size=16681) 2024-11-15T22:13:34,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:34,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708874221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:34,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:34,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:34,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708874235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:34,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:34,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708874236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:34,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:34,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708874236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:34,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:34,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708874237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:34,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:34,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:34,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:34,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708874346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:34,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:34,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708874346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:34,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:34,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:34,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:34,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708874358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:34,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:34,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708874358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:34,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:34,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:34,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708874358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:34,427 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T22:13:34,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:34,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708874566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:34,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:34,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708874567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:34,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:34,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:34,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:34,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708874577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:34,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:34,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/a0d3df1f399b4281a6c7f63aa53ba7a1 2024-11-15T22:13:34,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/ea424e6f9ef0484b8628a74120740f43 is 50, key is test_row_0/B:col10/1731708814161/Put/seqid=0 2024-11-15T22:13:34,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742308_1484 (size=12001) 2024-11-15T22:13:34,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:34,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708874586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:34,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:34,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708874586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:34,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:34,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708874876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:34,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708874876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:34,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708874897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:34,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:34,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:34,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708874906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:34,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708874906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:34,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:34,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:34,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:34,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/ea424e6f9ef0484b8628a74120740f43 2024-11-15T22:13:35,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/0f8217404bfb4c3ead03e146f9fb3410 is 50, key is test_row_0/C:col10/1731708814161/Put/seqid=0 2024-11-15T22:13:35,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742309_1485 (size=12001) 2024-11-15T22:13:35,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708875396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:35,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708875396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:35,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/0f8217404bfb4c3ead03e146f9fb3410 2024-11-15T22:13:35,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/a0d3df1f399b4281a6c7f63aa53ba7a1 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/a0d3df1f399b4281a6c7f63aa53ba7a1 2024-11-15T22:13:35,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/a0d3df1f399b4281a6c7f63aa53ba7a1, entries=250, sequenceid=52, filesize=16.3 K 2024-11-15T22:13:35,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/ea424e6f9ef0484b8628a74120740f43 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ea424e6f9ef0484b8628a74120740f43 2024-11-15T22:13:35,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ea424e6f9ef0484b8628a74120740f43, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T22:13:35,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/0f8217404bfb4c3ead03e146f9fb3410 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/0f8217404bfb4c3ead03e146f9fb3410 2024-11-15T22:13:35,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708875417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:35,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708875427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:35,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708875427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:35,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:35,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:35,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:35,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:35,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/0f8217404bfb4c3ead03e146f9fb3410, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T22:13:35,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for d0014c64eb4fbb0d0723a1402347f3ea in 1281ms, sequenceid=52, compaction requested=true 2024-11-15T22:13:35,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:35,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0014c64eb4fbb0d0723a1402347f3ea:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:13:35,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:35,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0014c64eb4fbb0d0723a1402347f3ea:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:13:35,461 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:35,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:35,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0014c64eb4fbb0d0723a1402347f3ea:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:13:35,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:35,461 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:35,463 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:35,463 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): d0014c64eb4fbb0d0723a1402347f3ea/A is initiating minor compaction (all files) 2024-11-15T22:13:35,463 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0014c64eb4fbb0d0723a1402347f3ea/A in TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:35,463 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/4b0bbbfe01f944d8815f761c4f4b6f37, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/85148d3424e04bd69960b414511ecbc7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/a0d3df1f399b4281a6c7f63aa53ba7a1] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp, totalSize=42.0 K 2024-11-15T22:13:35,464 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:35,464 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): d0014c64eb4fbb0d0723a1402347f3ea/B is initiating minor compaction (all files) 2024-11-15T22:13:35,464 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0014c64eb4fbb0d0723a1402347f3ea/B in TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:35,464 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/94cd9872ef5f429c8279e0fd6308cb8f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ffbb4bd2cadd433b932da698f1081af1, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ea424e6f9ef0484b8628a74120740f43] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp, totalSize=35.2 K 2024-11-15T22:13:35,464 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b0bbbfe01f944d8815f761c4f4b6f37, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731708811918 2024-11-15T22:13:35,465 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 94cd9872ef5f429c8279e0fd6308cb8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731708811920 2024-11-15T22:13:35,465 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85148d3424e04bd69960b414511ecbc7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731708811959 2024-11-15T22:13:35,466 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting ffbb4bd2cadd433b932da698f1081af1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731708811959 2024-11-15T22:13:35,466 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0d3df1f399b4281a6c7f63aa53ba7a1, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731708814161 2024-11-15T22:13:35,471 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting ea424e6f9ef0484b8628a74120740f43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731708814161 2024-11-15T22:13:35,488 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0014c64eb4fbb0d0723a1402347f3ea#A#compaction#395 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:35,488 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/a569be8b431146859fd8ab10b089dd42 is 50, key is test_row_0/A:col10/1731708814161/Put/seqid=0 2024-11-15T22:13:35,507 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0014c64eb4fbb0d0723a1402347f3ea#B#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:35,508 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/ae887d1f70024a19b5418865bed95eb0 is 50, key is test_row_0/B:col10/1731708814161/Put/seqid=0 2024-11-15T22:13:35,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742311_1487 (size=12104) 2024-11-15T22:13:35,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742310_1486 (size=12104) 2024-11-15T22:13:35,953 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/ae887d1f70024a19b5418865bed95eb0 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ae887d1f70024a19b5418865bed95eb0 2024-11-15T22:13:35,957 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/a569be8b431146859fd8ab10b089dd42 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/a569be8b431146859fd8ab10b089dd42 2024-11-15T22:13:35,961 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0014c64eb4fbb0d0723a1402347f3ea/B of d0014c64eb4fbb0d0723a1402347f3ea into ae887d1f70024a19b5418865bed95eb0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:35,961 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:35,961 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., storeName=d0014c64eb4fbb0d0723a1402347f3ea/B, priority=13, startTime=1731708815461; duration=0sec 2024-11-15T22:13:35,961 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:35,961 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0014c64eb4fbb0d0723a1402347f3ea:B 2024-11-15T22:13:35,961 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:35,966 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:35,966 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): d0014c64eb4fbb0d0723a1402347f3ea/C is initiating minor compaction (all files) 2024-11-15T22:13:35,966 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0014c64eb4fbb0d0723a1402347f3ea/C in TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:35,966 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/a535f1294611440cba285fa234080780, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/5cafb610329b4aa0b400033da1a45982, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/0f8217404bfb4c3ead03e146f9fb3410] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp, totalSize=35.2 K 2024-11-15T22:13:35,967 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting a535f1294611440cba285fa234080780, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731708811920 2024-11-15T22:13:35,967 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cafb610329b4aa0b400033da1a45982, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731708811959 2024-11-15T22:13:35,968 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f8217404bfb4c3ead03e146f9fb3410, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731708814161 2024-11-15T22:13:35,970 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0014c64eb4fbb0d0723a1402347f3ea/A of d0014c64eb4fbb0d0723a1402347f3ea into a569be8b431146859fd8ab10b089dd42(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:35,970 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:35,970 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., storeName=d0014c64eb4fbb0d0723a1402347f3ea/A, priority=13, startTime=1731708815461; duration=0sec 2024-11-15T22:13:35,970 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:35,970 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0014c64eb4fbb0d0723a1402347f3ea:A 2024-11-15T22:13:35,981 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0014c64eb4fbb0d0723a1402347f3ea#C#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:35,982 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/528b65322227404da662bf4f28b8812d is 50, key is test_row_0/C:col10/1731708814161/Put/seqid=0 2024-11-15T22:13:36,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742312_1488 (size=12104) 2024-11-15T22:13:36,012 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/528b65322227404da662bf4f28b8812d as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/528b65322227404da662bf4f28b8812d 2024-11-15T22:13:36,026 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0014c64eb4fbb0d0723a1402347f3ea/C of d0014c64eb4fbb0d0723a1402347f3ea into 528b65322227404da662bf4f28b8812d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:36,026 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:36,026 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., storeName=d0014c64eb4fbb0d0723a1402347f3ea/C, priority=13, startTime=1731708815461; duration=0sec 2024-11-15T22:13:36,026 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:36,026 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0014c64eb4fbb0d0723a1402347f3ea:C 2024-11-15T22:13:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T22:13:36,077 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:36,078 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:13:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-15T22:13:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T22:13:36,080 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:13:36,082 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:13:36,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:13:36,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T22:13:36,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T22:13:36,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:36,237 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing d0014c64eb4fbb0d0723a1402347f3ea 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T22:13:36,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/12f9191635e344779f1f2fcccff39cc5 is 50, key is test_row_0/A:col10/1731708814235/Put/seqid=0 2024-11-15T22:13:36,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742313_1489 (size=12001) 2024-11-15T22:13:36,283 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/12f9191635e344779f1f2fcccff39cc5 2024-11-15T22:13:36,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/fd13285cdfd54bb1b823d986889e18c8 is 50, key is test_row_0/B:col10/1731708814235/Put/seqid=0 2024-11-15T22:13:36,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742314_1490 (size=12001) 2024-11-15T22:13:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T22:13:36,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:36,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708876448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:36,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708876461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:36,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:36,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:36,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708876463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:36,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:36,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:36,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708876465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:36,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:36,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708876465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:36,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:36,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708876576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:36,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708876576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:36,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708876590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708876590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:36,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708876590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:36,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:36,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T22:13:36,727 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/fd13285cdfd54bb1b823d986889e18c8 2024-11-15T22:13:36,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/ff3547ea1ef14f56b79e51c941f604ee is 50, key is test_row_0/C:col10/1731708814235/Put/seqid=0 2024-11-15T22:13:36,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742315_1491 (size=12001) 2024-11-15T22:13:36,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708876796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:36,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708876796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:36,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708876797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:36,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708876797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:36,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708876807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:36,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:36,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:37,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708877116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:37,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:37,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708877117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:37,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:37,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:37,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708877117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:37,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:37,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:37,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708877117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:37,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:37,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:37,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708877127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:37,142 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/ff3547ea1ef14f56b79e51c941f604ee 2024-11-15T22:13:37,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/12f9191635e344779f1f2fcccff39cc5 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/12f9191635e344779f1f2fcccff39cc5 2024-11-15T22:13:37,156 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/12f9191635e344779f1f2fcccff39cc5, entries=150, sequenceid=78, filesize=11.7 K 2024-11-15T22:13:37,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/fd13285cdfd54bb1b823d986889e18c8 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/fd13285cdfd54bb1b823d986889e18c8 2024-11-15T22:13:37,170 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/fd13285cdfd54bb1b823d986889e18c8, entries=150, sequenceid=78, filesize=11.7 K 2024-11-15T22:13:37,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/ff3547ea1ef14f56b79e51c941f604ee as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/ff3547ea1ef14f56b79e51c941f604ee 2024-11-15T22:13:37,191 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/ff3547ea1ef14f56b79e51c941f604ee, entries=150, sequenceid=78, filesize=11.7 K 2024-11-15T22:13:37,193 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for d0014c64eb4fbb0d0723a1402347f3ea in 956ms, sequenceid=78, compaction requested=false 2024-11-15T22:13:37,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:37,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:37,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-15T22:13:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-15T22:13:37,202 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-15T22:13:37,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1170 sec 2024-11-15T22:13:37,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.1260 sec 2024-11-15T22:13:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T22:13:37,227 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:37,228 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:13:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-15T22:13:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T22:13:37,230 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:13:37,231 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:13:37,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:13:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T22:13:37,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T22:13:37,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:37,384 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing d0014c64eb4fbb0d0723a1402347f3ea 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T22:13:37,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/68d43b6d8cd34d0cb1df3e7de5647451 is 50, key is test_row_0/A:col10/1731708816463/Put/seqid=0 2024-11-15T22:13:37,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742316_1492 (size=12001) 2024-11-15T22:13:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T22:13:37,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:37,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:37,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:37,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708877698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:37,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:37,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708877698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:37,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708877699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708877700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708877701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:37,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708877817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:37,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708877826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:37,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708877826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708877826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708877826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:37,833 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/68d43b6d8cd34d0cb1df3e7de5647451 2024-11-15T22:13:37,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:37,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:37,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/c3dc5a8d7c1a4f5a94cb6d03a278006c is 50, key is test_row_0/B:col10/1731708816463/Put/seqid=0 2024-11-15T22:13:37,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742317_1493 (size=12001) 2024-11-15T22:13:37,845 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/c3dc5a8d7c1a4f5a94cb6d03a278006c 2024-11-15T22:13:37,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/5d2b55fc2569445f88152caccba6c176 is 50, key is test_row_0/C:col10/1731708816463/Put/seqid=0 2024-11-15T22:13:37,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742318_1494 (size=12001) 2024-11-15T22:13:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T22:13:38,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708878036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:38,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708878036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708878036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:38,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708878037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708878037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:38,256 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/5d2b55fc2569445f88152caccba6c176 2024-11-15T22:13:38,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/68d43b6d8cd34d0cb1df3e7de5647451 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/68d43b6d8cd34d0cb1df3e7de5647451 2024-11-15T22:13:38,266 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/68d43b6d8cd34d0cb1df3e7de5647451, entries=150, sequenceid=92, filesize=11.7 K 2024-11-15T22:13:38,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/c3dc5a8d7c1a4f5a94cb6d03a278006c as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/c3dc5a8d7c1a4f5a94cb6d03a278006c 2024-11-15T22:13:38,271 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/c3dc5a8d7c1a4f5a94cb6d03a278006c, entries=150, sequenceid=92, filesize=11.7 K 2024-11-15T22:13:38,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/5d2b55fc2569445f88152caccba6c176 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/5d2b55fc2569445f88152caccba6c176 2024-11-15T22:13:38,283 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/5d2b55fc2569445f88152caccba6c176, entries=150, sequenceid=92, filesize=11.7 K 2024-11-15T22:13:38,284 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d0014c64eb4fbb0d0723a1402347f3ea in 900ms, sequenceid=92, compaction requested=true 2024-11-15T22:13:38,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:38,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:38,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-15T22:13:38,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-15T22:13:38,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-15T22:13:38,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0550 sec 2024-11-15T22:13:38,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.0600 sec 2024-11-15T22:13:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:38,354 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:38,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0014c64eb4fbb0d0723a1402347f3ea 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T22:13:38,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/9531bf7f19c946eb866659aa5b4c1a62 is 50, key is test_row_0/A:col10/1731708817698/Put/seqid=0 2024-11-15T22:13:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T22:13:38,367 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:38,367 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:13:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-15T22:13:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T22:13:38,370 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:13:38,374 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:13:38,374 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:13:38,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742319_1495 (size=14341) 2024-11-15T22:13:38,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708878375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:38,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:38,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708878388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:38,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708878389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:38,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708878391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:38,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:38,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708878393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:38,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T22:13:38,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:38,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708878496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:38,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708878517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:38,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708878517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:38,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708878517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:38,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:38,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708878518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:38,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T22:13:38,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:38,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:38,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:38,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:38,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:38,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T22:13:38,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T22:13:38,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:38,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:38,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:38,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:38,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:38,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:38,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708878707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:38,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:38,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708878737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:38,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708878738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:38,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708878741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:38,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:38,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:38,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708878741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:38,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/9531bf7f19c946eb866659aa5b4c1a62 2024-11-15T22:13:38,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/e507f49366f24e83a6781549bb4d4ea0 is 50, key is test_row_0/B:col10/1731708817698/Put/seqid=0 2024-11-15T22:13:38,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742320_1496 (size=12001) 2024-11-15T22:13:38,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T22:13:38,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:38,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:38,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:38,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:38,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:38,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T22:13:39,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T22:13:39,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:39,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:39,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:39,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:39,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:39,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:39,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:39,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708879029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:39,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:39,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:39,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:39,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:39,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708879057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:39,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:39,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708879057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:39,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:39,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:39,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:39,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:39,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708879057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:39,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:39,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:39,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:39,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:39,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708879057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:39,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T22:13:39,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:39,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:39,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:39,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:39,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:39,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/e507f49366f24e83a6781549bb4d4ea0 2024-11-15T22:13:39,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/8251c79ee6944251af3dbffde5afcfb7 is 50, key is test_row_0/C:col10/1731708817698/Put/seqid=0 2024-11-15T22:13:39,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742321_1497 (size=12001) 2024-11-15T22:13:39,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T22:13:39,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:39,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:39,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:39,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:39,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:39,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T22:13:39,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:39,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:39,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:39,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:39,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T22:13:39,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708879547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:39,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:39,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:39,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:39,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708879576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:39,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708879577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:39,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708879577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:39,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708879577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:39,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T22:13:39,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:39,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:39,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:39,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:39,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:39,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/8251c79ee6944251af3dbffde5afcfb7 2024-11-15T22:13:39,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/9531bf7f19c946eb866659aa5b4c1a62 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/9531bf7f19c946eb866659aa5b4c1a62 2024-11-15T22:13:39,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/9531bf7f19c946eb866659aa5b4c1a62, entries=200, sequenceid=115, filesize=14.0 K 2024-11-15T22:13:39,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/e507f49366f24e83a6781549bb4d4ea0 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/e507f49366f24e83a6781549bb4d4ea0 2024-11-15T22:13:39,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/e507f49366f24e83a6781549bb4d4ea0, entries=150, sequenceid=115, filesize=11.7 K 2024-11-15T22:13:39,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/8251c79ee6944251af3dbffde5afcfb7 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/8251c79ee6944251af3dbffde5afcfb7 2024-11-15T22:13:39,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/8251c79ee6944251af3dbffde5afcfb7, entries=150, sequenceid=115, filesize=11.7 K 2024-11-15T22:13:39,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for d0014c64eb4fbb0d0723a1402347f3ea in 1360ms, sequenceid=115, compaction requested=true 2024-11-15T22:13:39,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:39,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0014c64eb4fbb0d0723a1402347f3ea:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:13:39,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:39,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0014c64eb4fbb0d0723a1402347f3ea:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:13:39,715 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:13:39,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:39,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0014c64eb4fbb0d0723a1402347f3ea:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:13:39,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:39,715 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:13:39,716 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:13:39,716 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): d0014c64eb4fbb0d0723a1402347f3ea/B is initiating minor compaction (all files) 2024-11-15T22:13:39,716 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0014c64eb4fbb0d0723a1402347f3ea/B in TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:39,716 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ae887d1f70024a19b5418865bed95eb0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/fd13285cdfd54bb1b823d986889e18c8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/c3dc5a8d7c1a4f5a94cb6d03a278006c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/e507f49366f24e83a6781549bb4d4ea0] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp, totalSize=47.0 K 2024-11-15T22:13:39,716 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:13:39,716 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): d0014c64eb4fbb0d0723a1402347f3ea/A is initiating minor compaction (all files) 2024-11-15T22:13:39,716 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0014c64eb4fbb0d0723a1402347f3ea/A in TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:39,716 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/a569be8b431146859fd8ab10b089dd42, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/12f9191635e344779f1f2fcccff39cc5, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/68d43b6d8cd34d0cb1df3e7de5647451, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/9531bf7f19c946eb866659aa5b4c1a62] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp, totalSize=49.3 K 2024-11-15T22:13:39,717 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting a569be8b431146859fd8ab10b089dd42, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731708814161 2024-11-15T22:13:39,717 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting ae887d1f70024a19b5418865bed95eb0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731708814161 2024-11-15T22:13:39,718 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12f9191635e344779f1f2fcccff39cc5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731708814234 2024-11-15T22:13:39,718 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting fd13285cdfd54bb1b823d986889e18c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731708814234 2024-11-15T22:13:39,718 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68d43b6d8cd34d0cb1df3e7de5647451, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731708816460 2024-11-15T22:13:39,718 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting c3dc5a8d7c1a4f5a94cb6d03a278006c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731708816460 2024-11-15T22:13:39,719 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9531bf7f19c946eb866659aa5b4c1a62, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731708817673 2024-11-15T22:13:39,719 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting e507f49366f24e83a6781549bb4d4ea0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731708817696 2024-11-15T22:13:39,729 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0014c64eb4fbb0d0723a1402347f3ea#B#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:39,730 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/3592a53a894243bb820bd0ac3e10e16c is 50, key is test_row_0/B:col10/1731708817698/Put/seqid=0 2024-11-15T22:13:39,731 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0014c64eb4fbb0d0723a1402347f3ea#A#compaction#408 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:39,731 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/ca3efac236794637a545fd66e1809902 is 50, key is test_row_0/A:col10/1731708817698/Put/seqid=0 2024-11-15T22:13:39,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742322_1498 (size=12241) 2024-11-15T22:13:39,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742323_1499 (size=12241) 2024-11-15T22:13:39,761 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/ca3efac236794637a545fd66e1809902 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/ca3efac236794637a545fd66e1809902 2024-11-15T22:13:39,768 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0014c64eb4fbb0d0723a1402347f3ea/A of d0014c64eb4fbb0d0723a1402347f3ea into ca3efac236794637a545fd66e1809902(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:39,768 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:39,768 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., storeName=d0014c64eb4fbb0d0723a1402347f3ea/A, priority=12, startTime=1731708819714; duration=0sec 2024-11-15T22:13:39,768 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:39,768 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0014c64eb4fbb0d0723a1402347f3ea:A 2024-11-15T22:13:39,769 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:13:39,770 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:13:39,770 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): d0014c64eb4fbb0d0723a1402347f3ea/C is initiating minor compaction (all files) 2024-11-15T22:13:39,770 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0014c64eb4fbb0d0723a1402347f3ea/C in TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:39,770 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/528b65322227404da662bf4f28b8812d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/ff3547ea1ef14f56b79e51c941f604ee, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/5d2b55fc2569445f88152caccba6c176, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/8251c79ee6944251af3dbffde5afcfb7] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp, totalSize=47.0 K 2024-11-15T22:13:39,770 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 528b65322227404da662bf4f28b8812d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731708814161 2024-11-15T22:13:39,771 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff3547ea1ef14f56b79e51c941f604ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731708814234 2024-11-15T22:13:39,773 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d2b55fc2569445f88152caccba6c176, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731708816460 2024-11-15T22:13:39,773 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8251c79ee6944251af3dbffde5afcfb7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731708817696 2024-11-15T22:13:39,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T22:13:39,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:39,780 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing d0014c64eb4fbb0d0723a1402347f3ea 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T22:13:39,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/868e0b2476d54692a77ce6ebd2c5dea7 is 50, key is test_row_0/A:col10/1731708818369/Put/seqid=0 2024-11-15T22:13:39,789 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0014c64eb4fbb0d0723a1402347f3ea#C#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:39,790 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/e49f1eea1bbf471c8ab40c0ba56886d0 is 50, key is test_row_0/C:col10/1731708817698/Put/seqid=0 2024-11-15T22:13:39,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742325_1501 (size=12241) 2024-11-15T22:13:39,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742324_1500 (size=12001) 2024-11-15T22:13:39,819 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/e49f1eea1bbf471c8ab40c0ba56886d0 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/e49f1eea1bbf471c8ab40c0ba56886d0 2024-11-15T22:13:39,828 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0014c64eb4fbb0d0723a1402347f3ea/C of d0014c64eb4fbb0d0723a1402347f3ea into e49f1eea1bbf471c8ab40c0ba56886d0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:39,828 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:39,828 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., storeName=d0014c64eb4fbb0d0723a1402347f3ea/C, priority=12, startTime=1731708819715; duration=0sec 2024-11-15T22:13:39,828 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:39,828 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0014c64eb4fbb0d0723a1402347f3ea:C 2024-11-15T22:13:40,154 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/3592a53a894243bb820bd0ac3e10e16c as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/3592a53a894243bb820bd0ac3e10e16c 2024-11-15T22:13:40,166 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0014c64eb4fbb0d0723a1402347f3ea/B of d0014c64eb4fbb0d0723a1402347f3ea into 3592a53a894243bb820bd0ac3e10e16c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:40,166 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:40,166 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., storeName=d0014c64eb4fbb0d0723a1402347f3ea/B, priority=12, startTime=1731708819715; duration=0sec 2024-11-15T22:13:40,166 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:40,166 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0014c64eb4fbb0d0723a1402347f3ea:B 2024-11-15T22:13:40,211 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/868e0b2476d54692a77ce6ebd2c5dea7 2024-11-15T22:13:40,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/b3a7ba64c91e4f1eb80896fa59f1fa1b is 50, key is test_row_0/B:col10/1731708818369/Put/seqid=0 2024-11-15T22:13:40,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742326_1502 (size=12001) 2024-11-15T22:13:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T22:13:40,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:40,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:40,622 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/b3a7ba64c91e4f1eb80896fa59f1fa1b 2024-11-15T22:13:40,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/aa889b1183264d2eb3467bbcce564597 is 50, key is test_row_0/C:col10/1731708818369/Put/seqid=0 2024-11-15T22:13:40,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742327_1503 (size=12001) 2024-11-15T22:13:40,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:40,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708880655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:40,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708880657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:40,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708880667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:40,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:40,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708880668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:40,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:40,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708880670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:40,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:40,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708880776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:40,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708880776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:40,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:40,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:40,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708880786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:40,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708880786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:40,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708880786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:40,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:40,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708880986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:40,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708880996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:40,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708881006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708881006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708881007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,032 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/aa889b1183264d2eb3467bbcce564597 2024-11-15T22:13:41,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/868e0b2476d54692a77ce6ebd2c5dea7 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/868e0b2476d54692a77ce6ebd2c5dea7 2024-11-15T22:13:41,039 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/868e0b2476d54692a77ce6ebd2c5dea7, entries=150, sequenceid=130, filesize=11.7 K 2024-11-15T22:13:41,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/b3a7ba64c91e4f1eb80896fa59f1fa1b as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/b3a7ba64c91e4f1eb80896fa59f1fa1b 2024-11-15T22:13:41,043 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/b3a7ba64c91e4f1eb80896fa59f1fa1b, entries=150, sequenceid=130, filesize=11.7 K 2024-11-15T22:13:41,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/aa889b1183264d2eb3467bbcce564597 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/aa889b1183264d2eb3467bbcce564597 2024-11-15T22:13:41,048 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/aa889b1183264d2eb3467bbcce564597, entries=150, sequenceid=130, filesize=11.7 K 2024-11-15T22:13:41,049 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for d0014c64eb4fbb0d0723a1402347f3ea in 1269ms, sequenceid=130, compaction requested=false 2024-11-15T22:13:41,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:41,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:41,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-15T22:13:41,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-15T22:13:41,052 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-15T22:13:41,052 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6760 sec 2024-11-15T22:13:41,055 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.6860 sec 2024-11-15T22:13:41,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:41,312 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:41,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0014c64eb4fbb0d0723a1402347f3ea 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T22:13:41,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/7bf138930aa64f178502fe055887f0cd is 50, key is test_row_0/A:col10/1731708821311/Put/seqid=0 2024-11-15T22:13:41,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742328_1504 (size=14541) 2024-11-15T22:13:41,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708881326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708881327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708881328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708881341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708881343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708881446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708881446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708881446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708881456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708881467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708881656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708881656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708881666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708881677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708881677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/7bf138930aa64f178502fe055887f0cd 2024-11-15T22:13:41,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/3213d7bb121b4deb848f085d0fd267ad is 50, key is test_row_0/B:col10/1731708821311/Put/seqid=0 2024-11-15T22:13:41,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742329_1505 (size=12151) 2024-11-15T22:13:41,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708881966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708881976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708881987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:41,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708881988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:42,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708881997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:42,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:42,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:42,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:42,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/3213d7bb121b4deb848f085d0fd267ad 2024-11-15T22:13:42,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/ac1a09a4b67a4ce7b6b588474be3c6af is 50, key is test_row_0/C:col10/1731708821311/Put/seqid=0 2024-11-15T22:13:42,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742330_1506 (size=12151) 2024-11-15T22:13:42,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708882486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:42,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:42,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:42,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708882497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:42,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:42,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:42,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:42,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:42,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708882506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:42,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:42,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:42,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:42,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708882506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:42,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:42,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:42,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:42,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:42,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:42,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708882516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:42,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:42,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:42,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T22:13:42,537 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:42,537 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:13:42,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-15T22:13:42,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T22:13:42,539 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:13:42,540 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:13:42,540 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:13:42,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/ac1a09a4b67a4ce7b6b588474be3c6af 2024-11-15T22:13:42,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/7bf138930aa64f178502fe055887f0cd as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/7bf138930aa64f178502fe055887f0cd 2024-11-15T22:13:42,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/7bf138930aa64f178502fe055887f0cd, entries=200, sequenceid=156, filesize=14.2 K 2024-11-15T22:13:42,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/3213d7bb121b4deb848f085d0fd267ad as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/3213d7bb121b4deb848f085d0fd267ad 2024-11-15T22:13:42,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/3213d7bb121b4deb848f085d0fd267ad, entries=150, sequenceid=156, filesize=11.9 K 2024-11-15T22:13:42,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/ac1a09a4b67a4ce7b6b588474be3c6af as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/ac1a09a4b67a4ce7b6b588474be3c6af 2024-11-15T22:13:42,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/ac1a09a4b67a4ce7b6b588474be3c6af, entries=150, sequenceid=156, filesize=11.9 K 2024-11-15T22:13:42,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for d0014c64eb4fbb0d0723a1402347f3ea in 1267ms, sequenceid=156, compaction requested=true 2024-11-15T22:13:42,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:42,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0014c64eb4fbb0d0723a1402347f3ea:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:13:42,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:42,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0014c64eb4fbb0d0723a1402347f3ea:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:13:42,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:42,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0014c64eb4fbb0d0723a1402347f3ea:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:13:42,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:42,579 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:42,579 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:42,580 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:42,580 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:42,580 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): d0014c64eb4fbb0d0723a1402347f3ea/A is initiating minor compaction (all files) 2024-11-15T22:13:42,580 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): d0014c64eb4fbb0d0723a1402347f3ea/B is initiating minor compaction (all files) 2024-11-15T22:13:42,580 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0014c64eb4fbb0d0723a1402347f3ea/A in TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:42,580 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0014c64eb4fbb0d0723a1402347f3ea/B in TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:42,580 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/ca3efac236794637a545fd66e1809902, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/868e0b2476d54692a77ce6ebd2c5dea7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/7bf138930aa64f178502fe055887f0cd] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp, totalSize=37.9 K 2024-11-15T22:13:42,580 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/3592a53a894243bb820bd0ac3e10e16c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/b3a7ba64c91e4f1eb80896fa59f1fa1b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/3213d7bb121b4deb848f085d0fd267ad] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp, totalSize=35.5 K 2024-11-15T22:13:42,581 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca3efac236794637a545fd66e1809902, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731708817696 2024-11-15T22:13:42,581 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 3592a53a894243bb820bd0ac3e10e16c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731708817696 2024-11-15T22:13:42,581 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting b3a7ba64c91e4f1eb80896fa59f1fa1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731708818369 2024-11-15T22:13:42,581 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 868e0b2476d54692a77ce6ebd2c5dea7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731708818369 2024-11-15T22:13:42,581 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 3213d7bb121b4deb848f085d0fd267ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731708820652 2024-11-15T22:13:42,581 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bf138930aa64f178502fe055887f0cd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731708820652 2024-11-15T22:13:42,594 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0014c64eb4fbb0d0723a1402347f3ea#B#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:42,594 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/625c18e8c33e41e49a4a77ac7f798602 is 50, key is test_row_0/B:col10/1731708821311/Put/seqid=0 2024-11-15T22:13:42,596 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0014c64eb4fbb0d0723a1402347f3ea#A#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:42,596 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/e767ea25ec4d416a8863b8713c31cadd is 50, key is test_row_0/A:col10/1731708821311/Put/seqid=0 2024-11-15T22:13:42,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742331_1507 (size=12493) 2024-11-15T22:13:42,609 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/625c18e8c33e41e49a4a77ac7f798602 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/625c18e8c33e41e49a4a77ac7f798602 2024-11-15T22:13:42,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742332_1508 (size=12493) 2024-11-15T22:13:42,614 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0014c64eb4fbb0d0723a1402347f3ea/B of d0014c64eb4fbb0d0723a1402347f3ea into 625c18e8c33e41e49a4a77ac7f798602(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:42,614 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:42,614 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., storeName=d0014c64eb4fbb0d0723a1402347f3ea/B, priority=13, startTime=1731708822579; duration=0sec 2024-11-15T22:13:42,614 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:42,614 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0014c64eb4fbb0d0723a1402347f3ea:B 2024-11-15T22:13:42,614 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:42,615 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:42,615 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): d0014c64eb4fbb0d0723a1402347f3ea/C is initiating minor compaction (all files) 2024-11-15T22:13:42,615 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0014c64eb4fbb0d0723a1402347f3ea/C in TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:42,615 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/e49f1eea1bbf471c8ab40c0ba56886d0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/aa889b1183264d2eb3467bbcce564597, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/ac1a09a4b67a4ce7b6b588474be3c6af] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp, totalSize=35.5 K 2024-11-15T22:13:42,616 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting e49f1eea1bbf471c8ab40c0ba56886d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731708817696 2024-11-15T22:13:42,616 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting aa889b1183264d2eb3467bbcce564597, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731708818369 2024-11-15T22:13:42,617 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting ac1a09a4b67a4ce7b6b588474be3c6af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731708820652 2024-11-15T22:13:42,622 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0014c64eb4fbb0d0723a1402347f3ea#C#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:42,622 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/c2ed1f7cb40344cab2cf72b1d6ae66c7 is 50, key is test_row_0/C:col10/1731708821311/Put/seqid=0 2024-11-15T22:13:42,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742333_1509 (size=12493) 2024-11-15T22:13:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T22:13:42,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T22:13:42,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:42,694 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing d0014c64eb4fbb0d0723a1402347f3ea 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T22:13:42,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/455adcf24b144fe7aca7a7c2dd2d5f7c is 50, key is test_row_0/A:col10/1731708821340/Put/seqid=0 2024-11-15T22:13:42,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742334_1510 (size=12151) 2024-11-15T22:13:42,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T22:13:43,015 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/e767ea25ec4d416a8863b8713c31cadd as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/e767ea25ec4d416a8863b8713c31cadd 2024-11-15T22:13:43,021 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0014c64eb4fbb0d0723a1402347f3ea/A of d0014c64eb4fbb0d0723a1402347f3ea into e767ea25ec4d416a8863b8713c31cadd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:43,022 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:43,022 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., storeName=d0014c64eb4fbb0d0723a1402347f3ea/A, priority=13, startTime=1731708822579; duration=0sec 2024-11-15T22:13:43,022 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:43,022 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0014c64eb4fbb0d0723a1402347f3ea:A 2024-11-15T22:13:43,044 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/c2ed1f7cb40344cab2cf72b1d6ae66c7 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/c2ed1f7cb40344cab2cf72b1d6ae66c7 2024-11-15T22:13:43,068 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0014c64eb4fbb0d0723a1402347f3ea/C of d0014c64eb4fbb0d0723a1402347f3ea into c2ed1f7cb40344cab2cf72b1d6ae66c7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:43,068 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:43,068 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., storeName=d0014c64eb4fbb0d0723a1402347f3ea/C, priority=13, startTime=1731708822579; duration=0sec 2024-11-15T22:13:43,068 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:43,068 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0014c64eb4fbb0d0723a1402347f3ea:C 2024-11-15T22:13:43,126 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/455adcf24b144fe7aca7a7c2dd2d5f7c 2024-11-15T22:13:43,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/126e07ae578d4392b22a9020b8c4b03d is 50, key is test_row_0/B:col10/1731708821340/Put/seqid=0 2024-11-15T22:13:43,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742335_1511 (size=12151) 2024-11-15T22:13:43,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T22:13:43,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:43,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:43,540 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/126e07ae578d4392b22a9020b8c4b03d 2024-11-15T22:13:43,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/7c844ed36a314b15b391fbe11ba5c2e5 is 50, key is test_row_0/C:col10/1731708821340/Put/seqid=0 2024-11-15T22:13:43,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:43,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708883554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:43,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:43,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708883560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:43,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:43,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708883563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:43,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:43,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708883563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:43,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:43,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742336_1512 (size=12151) 2024-11-15T22:13:43,581 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/7c844ed36a314b15b391fbe11ba5c2e5 2024-11-15T22:13:43,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:43,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708883573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:43,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:43,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/455adcf24b144fe7aca7a7c2dd2d5f7c as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/455adcf24b144fe7aca7a7c2dd2d5f7c 2024-11-15T22:13:43,593 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/455adcf24b144fe7aca7a7c2dd2d5f7c, entries=150, sequenceid=169, filesize=11.9 K 2024-11-15T22:13:43,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/126e07ae578d4392b22a9020b8c4b03d as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/126e07ae578d4392b22a9020b8c4b03d 2024-11-15T22:13:43,602 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/126e07ae578d4392b22a9020b8c4b03d, entries=150, sequenceid=169, filesize=11.9 K 2024-11-15T22:13:43,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/7c844ed36a314b15b391fbe11ba5c2e5 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/7c844ed36a314b15b391fbe11ba5c2e5 2024-11-15T22:13:43,613 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/7c844ed36a314b15b391fbe11ba5c2e5, entries=150, sequenceid=169, filesize=11.9 K 2024-11-15T22:13:43,614 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for d0014c64eb4fbb0d0723a1402347f3ea in 920ms, sequenceid=169, compaction requested=false 2024-11-15T22:13:43,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:43,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:43,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-15T22:13:43,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-15T22:13:43,619 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-15T22:13:43,619 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0780 sec 2024-11-15T22:13:43,622 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.0830 sec 2024-11-15T22:13:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T22:13:43,676 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:43,677 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:13:43,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-15T22:13:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:43,679 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:43,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0014c64eb4fbb0d0723a1402347f3ea 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T22:13:43,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T22:13:43,680 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:13:43,681 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:13:43,681 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:13:43,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/0ca667ace35b46cca83e5b590fb2a261 is 50, key is test_row_0/A:col10/1731708823549/Put/seqid=0 2024-11-15T22:13:43,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742337_1513 (size=14541) 2024-11-15T22:13:43,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708883703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:43,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:43,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708883704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:43,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:43,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708883704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:43,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:43,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708883713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:43,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708883714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T22:13:43,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708883816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:43,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708883827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:43,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708883827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:43,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708883827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:43,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708883827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:43,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:43,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T22:13:43,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:43,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:43,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:43,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:43,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:43,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:43,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:43,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:43,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T22:13:43,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:43,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:43,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:43,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:43,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:43,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T22:13:44,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708884027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:44,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:44,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708884036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:44,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708884036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:44,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708884036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:44,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708884046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:44,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:44,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/0ca667ace35b46cca83e5b590fb2a261 2024-11-15T22:13:44,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/b8324fa4627d495f966b581315d7ae58 is 50, key is test_row_0/B:col10/1731708823549/Put/seqid=0 2024-11-15T22:13:44,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742338_1514 (size=12151) 2024-11-15T22:13:44,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T22:13:44,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:44,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:44,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:44,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:44,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:44,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T22:13:44,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:44,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:44,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:44,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:44,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T22:13:44,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708884337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:44,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708884356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:44,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708884356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:44,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708884356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:44,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:44,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708884356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:44,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T22:13:44,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:44,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:44,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:44,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:44,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:44,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/b8324fa4627d495f966b581315d7ae58 2024-11-15T22:13:44,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/48d96f28751840778853f8f964a9b43a is 50, key is test_row_0/C:col10/1731708823549/Put/seqid=0 2024-11-15T22:13:44,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742339_1515 (size=12151) 2024-11-15T22:13:44,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T22:13:44,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:44,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:44,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:44,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:44,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:44,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T22:13:44,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:44,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:44,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:44,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:44,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:44,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T22:13:44,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:44,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708884857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:44,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:44,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:44,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708884866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:44,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:44,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708884866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:44,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:44,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708884866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:44,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:44,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708884876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:44,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T22:13:44,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:44,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:44,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:44,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:44,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:44,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:44,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/48d96f28751840778853f8f964a9b43a 2024-11-15T22:13:44,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/0ca667ace35b46cca83e5b590fb2a261 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/0ca667ace35b46cca83e5b590fb2a261 2024-11-15T22:13:44,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/0ca667ace35b46cca83e5b590fb2a261, entries=200, sequenceid=196, filesize=14.2 K 2024-11-15T22:13:44,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/b8324fa4627d495f966b581315d7ae58 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/b8324fa4627d495f966b581315d7ae58 2024-11-15T22:13:44,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/b8324fa4627d495f966b581315d7ae58, entries=150, sequenceid=196, filesize=11.9 K 2024-11-15T22:13:44,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/48d96f28751840778853f8f964a9b43a as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/48d96f28751840778853f8f964a9b43a 2024-11-15T22:13:44,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/48d96f28751840778853f8f964a9b43a, entries=150, sequenceid=196, filesize=11.9 K 2024-11-15T22:13:44,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d0014c64eb4fbb0d0723a1402347f3ea in 1278ms, sequenceid=196, compaction requested=true 2024-11-15T22:13:44,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:44,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0014c64eb4fbb0d0723a1402347f3ea:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:13:44,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:44,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0014c64eb4fbb0d0723a1402347f3ea:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:13:44,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:44,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0014c64eb4fbb0d0723a1402347f3ea:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:13:44,957 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:44,957 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:44,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:44,960 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:44,960 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:44,960 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): d0014c64eb4fbb0d0723a1402347f3ea/B is initiating minor compaction (all files) 2024-11-15T22:13:44,960 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): d0014c64eb4fbb0d0723a1402347f3ea/A is initiating minor compaction (all files) 2024-11-15T22:13:44,960 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0014c64eb4fbb0d0723a1402347f3ea/B in TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:44,960 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0014c64eb4fbb0d0723a1402347f3ea/A in TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:44,960 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/625c18e8c33e41e49a4a77ac7f798602, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/126e07ae578d4392b22a9020b8c4b03d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/b8324fa4627d495f966b581315d7ae58] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp, totalSize=35.9 K 2024-11-15T22:13:44,960 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/e767ea25ec4d416a8863b8713c31cadd, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/455adcf24b144fe7aca7a7c2dd2d5f7c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/0ca667ace35b46cca83e5b590fb2a261] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp, totalSize=38.3 K 2024-11-15T22:13:44,960 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 625c18e8c33e41e49a4a77ac7f798602, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731708820652 2024-11-15T22:13:44,960 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting e767ea25ec4d416a8863b8713c31cadd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731708820652 2024-11-15T22:13:44,961 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 455adcf24b144fe7aca7a7c2dd2d5f7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731708821326 2024-11-15T22:13:44,961 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 126e07ae578d4392b22a9020b8c4b03d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731708821326 2024-11-15T22:13:44,961 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting b8324fa4627d495f966b581315d7ae58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731708823549 2024-11-15T22:13:44,961 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ca667ace35b46cca83e5b590fb2a261, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731708823549 2024-11-15T22:13:44,967 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0014c64eb4fbb0d0723a1402347f3ea#A#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:44,967 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0014c64eb4fbb0d0723a1402347f3ea#B#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:44,968 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/ab6cc2e5f44c4f7d9552d9a6a2df6484 is 50, key is test_row_0/B:col10/1731708823549/Put/seqid=0 2024-11-15T22:13:44,968 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/f1754f21512e4c47ae626b63483a6ac2 is 50, key is test_row_0/A:col10/1731708823549/Put/seqid=0 2024-11-15T22:13:44,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742340_1516 (size=12595) 2024-11-15T22:13:44,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742341_1517 (size=12595) 2024-11-15T22:13:45,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T22:13:45,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:45,060 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing d0014c64eb4fbb0d0723a1402347f3ea 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T22:13:45,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/6fc77560dfe24a62909181e657ba5b2c is 50, key is test_row_0/A:col10/1731708823702/Put/seqid=0 2024-11-15T22:13:45,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742342_1518 (size=12151) 2024-11-15T22:13:45,377 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/f1754f21512e4c47ae626b63483a6ac2 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/f1754f21512e4c47ae626b63483a6ac2 2024-11-15T22:13:45,381 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/ab6cc2e5f44c4f7d9552d9a6a2df6484 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ab6cc2e5f44c4f7d9552d9a6a2df6484 2024-11-15T22:13:45,390 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0014c64eb4fbb0d0723a1402347f3ea/A of d0014c64eb4fbb0d0723a1402347f3ea into f1754f21512e4c47ae626b63483a6ac2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:45,391 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:45,391 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., storeName=d0014c64eb4fbb0d0723a1402347f3ea/A, priority=13, startTime=1731708824957; duration=0sec 2024-11-15T22:13:45,391 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:45,391 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0014c64eb4fbb0d0723a1402347f3ea:A 2024-11-15T22:13:45,391 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:45,391 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0014c64eb4fbb0d0723a1402347f3ea/B of d0014c64eb4fbb0d0723a1402347f3ea into ab6cc2e5f44c4f7d9552d9a6a2df6484(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:45,391 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:45,392 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., storeName=d0014c64eb4fbb0d0723a1402347f3ea/B, priority=13, startTime=1731708824957; duration=0sec 2024-11-15T22:13:45,392 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:45,392 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0014c64eb4fbb0d0723a1402347f3ea:B 2024-11-15T22:13:45,392 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:45,392 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): d0014c64eb4fbb0d0723a1402347f3ea/C is initiating minor compaction (all files) 2024-11-15T22:13:45,392 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0014c64eb4fbb0d0723a1402347f3ea/C in TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:45,392 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/c2ed1f7cb40344cab2cf72b1d6ae66c7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/7c844ed36a314b15b391fbe11ba5c2e5, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/48d96f28751840778853f8f964a9b43a] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp, totalSize=35.9 K 2024-11-15T22:13:45,393 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2ed1f7cb40344cab2cf72b1d6ae66c7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731708820652 2024-11-15T22:13:45,394 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c844ed36a314b15b391fbe11ba5c2e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731708821326 2024-11-15T22:13:45,395 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48d96f28751840778853f8f964a9b43a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731708823549 2024-11-15T22:13:45,401 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0014c64eb4fbb0d0723a1402347f3ea#C#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:45,401 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/7c881285956b4b378f2bc9039041f2d6 is 50, key is test_row_0/C:col10/1731708823549/Put/seqid=0 2024-11-15T22:13:45,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742343_1519 (size=12595) 2024-11-15T22:13:45,470 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/6fc77560dfe24a62909181e657ba5b2c 2024-11-15T22:13:45,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/419da4359a6b426d8228f8d41af33bea is 50, key is test_row_0/B:col10/1731708823702/Put/seqid=0 2024-11-15T22:13:45,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742344_1520 (size=12151) 2024-11-15T22:13:45,811 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/7c881285956b4b378f2bc9039041f2d6 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/7c881285956b4b378f2bc9039041f2d6 2024-11-15T22:13:45,815 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0014c64eb4fbb0d0723a1402347f3ea/C of d0014c64eb4fbb0d0723a1402347f3ea into 7c881285956b4b378f2bc9039041f2d6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:45,816 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:45,816 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., storeName=d0014c64eb4fbb0d0723a1402347f3ea/C, priority=13, startTime=1731708824957; duration=0sec 2024-11-15T22:13:45,816 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:45,816 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0014c64eb4fbb0d0723a1402347f3ea:C 2024-11-15T22:13:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T22:13:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:45,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:45,881 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/419da4359a6b426d8228f8d41af33bea 2024-11-15T22:13:45,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/c0449cf237d34b578a1163480ec61e20 is 50, key is test_row_0/C:col10/1731708823702/Put/seqid=0 2024-11-15T22:13:45,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742345_1521 (size=12151) 2024-11-15T22:13:45,893 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/c0449cf237d34b578a1163480ec61e20 2024-11-15T22:13:45,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/6fc77560dfe24a62909181e657ba5b2c as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/6fc77560dfe24a62909181e657ba5b2c 2024-11-15T22:13:45,900 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/6fc77560dfe24a62909181e657ba5b2c, entries=150, sequenceid=207, filesize=11.9 K 2024-11-15T22:13:45,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/419da4359a6b426d8228f8d41af33bea as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/419da4359a6b426d8228f8d41af33bea 2024-11-15T22:13:45,904 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/419da4359a6b426d8228f8d41af33bea, entries=150, sequenceid=207, filesize=11.9 K 2024-11-15T22:13:45,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/c0449cf237d34b578a1163480ec61e20 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/c0449cf237d34b578a1163480ec61e20 2024-11-15T22:13:45,908 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/c0449cf237d34b578a1163480ec61e20, entries=150, sequenceid=207, filesize=11.9 K 2024-11-15T22:13:45,908 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=73.80 KB/75570 for d0014c64eb4fbb0d0723a1402347f3ea in 848ms, sequenceid=207, compaction requested=false 2024-11-15T22:13:45,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:45,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:45,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-15T22:13:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-15T22:13:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:45,910 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:45,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0014c64eb4fbb0d0723a1402347f3ea 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T22:13:45,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-15T22:13:45,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2290 sec 2024-11-15T22:13:45,914 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 2.2360 sec 2024-11-15T22:13:45,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/597a55f1d0654fe28fb6c6ba610b2cb0 is 50, key is test_row_0/A:col10/1731708825900/Put/seqid=0 2024-11-15T22:13:45,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742346_1522 (size=19321) 2024-11-15T22:13:45,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708885935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:45,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708885936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:45,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708885937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:45,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:45,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708885944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:45,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708885944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:45,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:45,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:45,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:45,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:45,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:45,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:46,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708886046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:46,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708886057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:46,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:46,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708886057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:46,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:46,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708886066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:46,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:46,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708886066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:46,075 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:46,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:46,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708886267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:46,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708886267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:46,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:46,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708886267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:46,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:46,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:46,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708886286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:46,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708886287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:46,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:46,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/597a55f1d0654fe28fb6c6ba610b2cb0 2024-11-15T22:13:46,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/93c4d3f7ac1c4b538b318e8e11c12a8c is 50, key is test_row_0/B:col10/1731708825900/Put/seqid=0 2024-11-15T22:13:46,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742347_1523 (size=12151) 2024-11-15T22:13:46,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/93c4d3f7ac1c4b538b318e8e11c12a8c 2024-11-15T22:13:46,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/7b31880cb7d8461bb2c8c312a96564c6 is 50, key is test_row_0/C:col10/1731708825900/Put/seqid=0 2024-11-15T22:13:46,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742348_1524 (size=12151) 2024-11-15T22:13:46,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708886586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:46,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708886586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:46,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:46,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:46,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708886586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:46,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:46,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:46,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708886597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:46,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708886606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:46,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:46,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:46,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/7b31880cb7d8461bb2c8c312a96564c6 2024-11-15T22:13:46,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/597a55f1d0654fe28fb6c6ba610b2cb0 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/597a55f1d0654fe28fb6c6ba610b2cb0 2024-11-15T22:13:46,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/597a55f1d0654fe28fb6c6ba610b2cb0, entries=300, sequenceid=228, filesize=18.9 K 2024-11-15T22:13:46,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/93c4d3f7ac1c4b538b318e8e11c12a8c as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/93c4d3f7ac1c4b538b318e8e11c12a8c 2024-11-15T22:13:46,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/93c4d3f7ac1c4b538b318e8e11c12a8c, entries=150, sequenceid=228, filesize=11.9 K 2024-11-15T22:13:46,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/7b31880cb7d8461bb2c8c312a96564c6 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/7b31880cb7d8461bb2c8c312a96564c6 2024-11-15T22:13:46,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/7b31880cb7d8461bb2c8c312a96564c6, entries=150, sequenceid=228, filesize=11.9 K 2024-11-15T22:13:46,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d0014c64eb4fbb0d0723a1402347f3ea in 896ms, sequenceid=228, compaction requested=true 2024-11-15T22:13:46,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:46,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0014c64eb4fbb0d0723a1402347f3ea:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:13:46,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:46,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0014c64eb4fbb0d0723a1402347f3ea:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:13:46,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:46,806 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:46,806 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:46,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0014c64eb4fbb0d0723a1402347f3ea:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:13:46,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:46,807 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44067 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:46,807 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:46,807 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): d0014c64eb4fbb0d0723a1402347f3ea/A is initiating minor compaction (all files) 2024-11-15T22:13:46,807 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): d0014c64eb4fbb0d0723a1402347f3ea/B is initiating minor compaction (all files) 2024-11-15T22:13:46,807 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0014c64eb4fbb0d0723a1402347f3ea/B in TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:46,807 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0014c64eb4fbb0d0723a1402347f3ea/A in TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:46,807 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/f1754f21512e4c47ae626b63483a6ac2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/6fc77560dfe24a62909181e657ba5b2c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/597a55f1d0654fe28fb6c6ba610b2cb0] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp, totalSize=43.0 K 2024-11-15T22:13:46,807 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ab6cc2e5f44c4f7d9552d9a6a2df6484, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/419da4359a6b426d8228f8d41af33bea, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/93c4d3f7ac1c4b538b318e8e11c12a8c] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp, totalSize=36.0 K 2024-11-15T22:13:46,807 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1754f21512e4c47ae626b63483a6ac2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731708823549 2024-11-15T22:13:46,807 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting ab6cc2e5f44c4f7d9552d9a6a2df6484, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731708823549 2024-11-15T22:13:46,808 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fc77560dfe24a62909181e657ba5b2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731708823697 2024-11-15T22:13:46,808 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 419da4359a6b426d8228f8d41af33bea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731708823697 2024-11-15T22:13:46,808 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 597a55f1d0654fe28fb6c6ba610b2cb0, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731708825900 2024-11-15T22:13:46,808 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 93c4d3f7ac1c4b538b318e8e11c12a8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731708825900 2024-11-15T22:13:46,815 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0014c64eb4fbb0d0723a1402347f3ea#A#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:46,816 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/b4c4a66be45241aaa87a972b0f943f31 is 50, key is test_row_0/A:col10/1731708825900/Put/seqid=0 2024-11-15T22:13:46,817 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0014c64eb4fbb0d0723a1402347f3ea#B#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:46,817 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/21c08b533bb4480986616cb2e1b1e4bd is 50, key is test_row_0/B:col10/1731708825900/Put/seqid=0 2024-11-15T22:13:46,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742349_1525 (size=12697) 2024-11-15T22:13:46,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742350_1526 (size=12697) 2024-11-15T22:13:46,827 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/21c08b533bb4480986616cb2e1b1e4bd as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/21c08b533bb4480986616cb2e1b1e4bd 2024-11-15T22:13:46,831 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0014c64eb4fbb0d0723a1402347f3ea/B of d0014c64eb4fbb0d0723a1402347f3ea into 21c08b533bb4480986616cb2e1b1e4bd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:46,831 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:46,831 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., storeName=d0014c64eb4fbb0d0723a1402347f3ea/B, priority=13, startTime=1731708826806; duration=0sec 2024-11-15T22:13:46,831 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:46,831 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0014c64eb4fbb0d0723a1402347f3ea:B 2024-11-15T22:13:46,831 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:46,832 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:46,833 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): d0014c64eb4fbb0d0723a1402347f3ea/C is initiating minor compaction (all files) 2024-11-15T22:13:46,834 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0014c64eb4fbb0d0723a1402347f3ea/C in TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:46,834 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/7c881285956b4b378f2bc9039041f2d6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/c0449cf237d34b578a1163480ec61e20, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/7b31880cb7d8461bb2c8c312a96564c6] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp, totalSize=36.0 K 2024-11-15T22:13:46,834 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c881285956b4b378f2bc9039041f2d6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731708823549 2024-11-15T22:13:46,835 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting c0449cf237d34b578a1163480ec61e20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731708823697 2024-11-15T22:13:46,835 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b31880cb7d8461bb2c8c312a96564c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731708825900 2024-11-15T22:13:46,842 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0014c64eb4fbb0d0723a1402347f3ea#C#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:46,843 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/f2ac0186712848c892f0c4f69cd64320 is 50, key is test_row_0/C:col10/1731708825900/Put/seqid=0 2024-11-15T22:13:46,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742351_1527 (size=12697) 2024-11-15T22:13:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:47,113 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:47,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0014c64eb4fbb0d0723a1402347f3ea 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T22:13:47,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/31b68427263a4a9598e046a3a9753a7e is 50, key is test_row_0/A:col10/1731708827112/Put/seqid=0 2024-11-15T22:13:47,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742352_1528 (size=14541) 2024-11-15T22:13:47,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:47,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708887138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:47,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:47,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708887139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:47,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:47,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:47,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:47,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708887139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:47,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:47,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708887140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:47,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:47,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:47,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:47,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708887147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:47,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:47,224 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/b4c4a66be45241aaa87a972b0f943f31 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/b4c4a66be45241aaa87a972b0f943f31 2024-11-15T22:13:47,228 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0014c64eb4fbb0d0723a1402347f3ea/A of d0014c64eb4fbb0d0723a1402347f3ea into b4c4a66be45241aaa87a972b0f943f31(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:47,228 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:47,228 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., storeName=d0014c64eb4fbb0d0723a1402347f3ea/A, priority=13, startTime=1731708826806; duration=0sec 2024-11-15T22:13:47,228 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:47,228 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0014c64eb4fbb0d0723a1402347f3ea:A 2024-11-15T22:13:47,254 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/f2ac0186712848c892f0c4f69cd64320 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/f2ac0186712848c892f0c4f69cd64320 2024-11-15T22:13:47,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708887257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:47,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708887257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:47,260 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0014c64eb4fbb0d0723a1402347f3ea/C of d0014c64eb4fbb0d0723a1402347f3ea into f2ac0186712848c892f0c4f69cd64320(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:47,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:47,260 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708887258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:47,260 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., storeName=d0014c64eb4fbb0d0723a1402347f3ea/C, priority=13, startTime=1731708826806; duration=0sec 2024-11-15T22:13:47,260 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:47,260 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0014c64eb4fbb0d0723a1402347f3ea:C 2024-11-15T22:13:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:47,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:47,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708887266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:47,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708887266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:47,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708887467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:47,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708887467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708887467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:47,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708887476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:47,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708887476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:47,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/31b68427263a4a9598e046a3a9753a7e 2024-11-15T22:13:47,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/fd74490d45f741e580d9274239f7c662 is 50, key is test_row_0/B:col10/1731708827112/Put/seqid=0 2024-11-15T22:13:47,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742353_1529 (size=12151) 2024-11-15T22:13:47,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708887786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:47,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708887787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:47,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708887787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:47,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:47,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:47,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:47,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708887797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:47,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708887797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T22:13:47,847 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:47,847 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:13:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-15T22:13:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T22:13:47,851 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:13:47,851 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:13:47,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:13:47,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/fd74490d45f741e580d9274239f7c662 2024-11-15T22:13:47,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/864cc05cb1c34b51bb0dd0d0fb229dc7 is 50, key is test_row_0/C:col10/1731708827112/Put/seqid=0 2024-11-15T22:13:47,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742354_1530 (size=12151) 2024-11-15T22:13:47,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/864cc05cb1c34b51bb0dd0d0fb229dc7 2024-11-15T22:13:47,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/31b68427263a4a9598e046a3a9753a7e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/31b68427263a4a9598e046a3a9753a7e 2024-11-15T22:13:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T22:13:47,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/31b68427263a4a9598e046a3a9753a7e, entries=200, sequenceid=248, filesize=14.2 K 2024-11-15T22:13:47,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/fd74490d45f741e580d9274239f7c662 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/fd74490d45f741e580d9274239f7c662 2024-11-15T22:13:47,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/fd74490d45f741e580d9274239f7c662, entries=150, sequenceid=248, filesize=11.9 K 2024-11-15T22:13:47,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/864cc05cb1c34b51bb0dd0d0fb229dc7 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/864cc05cb1c34b51bb0dd0d0fb229dc7 2024-11-15T22:13:47,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/864cc05cb1c34b51bb0dd0d0fb229dc7, entries=150, sequenceid=248, filesize=11.9 K 2024-11-15T22:13:47,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for d0014c64eb4fbb0d0723a1402347f3ea in 853ms, sequenceid=248, compaction requested=false 2024-11-15T22:13:47,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:48,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-15T22:13:48,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:48,006 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing d0014c64eb4fbb0d0723a1402347f3ea 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T22:13:48,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/69f3ea57120f42c5a1160359d2ea702c is 50, key is test_row_0/A:col10/1731708827126/Put/seqid=0 2024-11-15T22:13:48,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742355_1531 (size=12301) 2024-11-15T22:13:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T22:13:48,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:48,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708888334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:48,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708888338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:48,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:48,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708888339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:48,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:48,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:48,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708888340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:48,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:48,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:48,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708888346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:48,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:48,414 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/69f3ea57120f42c5a1160359d2ea702c 2024-11-15T22:13:48,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/7911180183c549da912c85bce6680c6d is 50, key is test_row_0/B:col10/1731708827126/Put/seqid=0 2024-11-15T22:13:48,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742356_1532 (size=12301) 2024-11-15T22:13:48,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708888457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:48,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708888457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708888457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:48,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708888457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T22:13:48,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708888466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:48,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:48,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:48,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708888676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:48,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:48,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708888676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:48,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:48,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708888676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:48,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708888676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:48,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:48,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708888687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:48,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:48,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:48,826 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/7911180183c549da912c85bce6680c6d 2024-11-15T22:13:48,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/d9dfc1e666674e67a2fabca2a5eb8585 is 50, key is test_row_0/C:col10/1731708827126/Put/seqid=0 2024-11-15T22:13:48,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742357_1533 (size=12301) 2024-11-15T22:13:48,837 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/d9dfc1e666674e67a2fabca2a5eb8585 2024-11-15T22:13:48,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/69f3ea57120f42c5a1160359d2ea702c as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/69f3ea57120f42c5a1160359d2ea702c 2024-11-15T22:13:48,844 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/69f3ea57120f42c5a1160359d2ea702c, entries=150, sequenceid=267, filesize=12.0 K 2024-11-15T22:13:48,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/7911180183c549da912c85bce6680c6d as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/7911180183c549da912c85bce6680c6d 2024-11-15T22:13:48,848 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/7911180183c549da912c85bce6680c6d, entries=150, sequenceid=267, filesize=12.0 K 2024-11-15T22:13:48,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/d9dfc1e666674e67a2fabca2a5eb8585 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/d9dfc1e666674e67a2fabca2a5eb8585 2024-11-15T22:13:48,854 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/d9dfc1e666674e67a2fabca2a5eb8585, entries=150, sequenceid=267, filesize=12.0 K 2024-11-15T22:13:48,855 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for d0014c64eb4fbb0d0723a1402347f3ea in 849ms, sequenceid=267, compaction requested=true 2024-11-15T22:13:48,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:48,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:48,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-15T22:13:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-15T22:13:48,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-15T22:13:48,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0050 sec 2024-11-15T22:13:48,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.0120 sec 2024-11-15T22:13:48,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T22:13:48,986 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:48,987 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:13:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-15T22:13:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-15T22:13:48,989 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:13:48,989 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:13:48,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:13:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:48,990 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:48,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0014c64eb4fbb0d0723a1402347f3ea 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-15T22:13:48,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/9819d07e08a74d889a5abc1389029d65 is 50, key is test_row_0/A:col10/1731708828989/Put/seqid=0 2024-11-15T22:13:49,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742358_1534 (size=14741) 2024-11-15T22:13:49,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708889022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:49,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708889023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:49,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708889023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708889024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:49,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708889025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:49,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-15T22:13:49,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-15T22:13:49,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:49,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:49,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:49,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:49,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:49,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:49,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:49,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708889136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:49,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:49,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:49,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708889147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:49,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708889147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:49,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708889147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:49,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708889147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:49,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:49,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-15T22:13:49,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:49,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:49,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:49,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:49,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:49,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:49,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-15T22:13:49,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:49,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708889356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:49,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:49,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:49,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708889367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:49,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:49,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708889367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:49,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:49,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:49,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708889367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:49,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:49,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:49,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:49,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708889367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:49,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:49,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/9819d07e08a74d889a5abc1389029d65 2024-11-15T22:13:49,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/a890be84de38486eb3231870dc26e74e is 50, key is test_row_0/B:col10/1731708828989/Put/seqid=0 2024-11-15T22:13:49,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742359_1535 (size=12301) 2024-11-15T22:13:49,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-15T22:13:49,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:49,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:49,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:49,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:49,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:49,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:49,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-15T22:13:49,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:49,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:49,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:49,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:49,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-15T22:13:49,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708889676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:49,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:49,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708889686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:49,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708889686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:49,697 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,697 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,697 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:49,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:49,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708889686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:49,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:49,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:49,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708889687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:49,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:49,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:49,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:49,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-15T22:13:49,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:49,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:49,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:49,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:49,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:49,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:13:49,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/a890be84de38486eb3231870dc26e74e 2024-11-15T22:13:49,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/79a36ab418414fa093d33d3d87162341 is 50, key is test_row_0/C:col10/1731708828989/Put/seqid=0 2024-11-15T22:13:49,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742360_1536 (size=12301) 2024-11-15T22:13:49,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/79a36ab418414fa093d33d3d87162341 2024-11-15T22:13:49,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/9819d07e08a74d889a5abc1389029d65 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/9819d07e08a74d889a5abc1389029d65 2024-11-15T22:13:49,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/9819d07e08a74d889a5abc1389029d65, entries=200, sequenceid=287, filesize=14.4 K 2024-11-15T22:13:49,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/a890be84de38486eb3231870dc26e74e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/a890be84de38486eb3231870dc26e74e 2024-11-15T22:13:49,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/a890be84de38486eb3231870dc26e74e, entries=150, sequenceid=287, filesize=12.0 K 2024-11-15T22:13:49,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/79a36ab418414fa093d33d3d87162341 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/79a36ab418414fa093d33d3d87162341 2024-11-15T22:13:49,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/79a36ab418414fa093d33d3d87162341, entries=150, sequenceid=287, filesize=12.0 K 2024-11-15T22:13:49,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for d0014c64eb4fbb0d0723a1402347f3ea in 864ms, sequenceid=287, compaction requested=true 2024-11-15T22:13:49,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:49,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0014c64eb4fbb0d0723a1402347f3ea:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:13:49,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:49,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0014c64eb4fbb0d0723a1402347f3ea:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:13:49,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:49,855 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:13:49,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0014c64eb4fbb0d0723a1402347f3ea:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:13:49,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:49,855 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:13:49,859 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:13:49,859 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54280 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:13:49,859 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): d0014c64eb4fbb0d0723a1402347f3ea/B is initiating minor compaction (all files) 2024-11-15T22:13:49,859 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): d0014c64eb4fbb0d0723a1402347f3ea/A is initiating minor compaction (all files) 2024-11-15T22:13:49,859 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0014c64eb4fbb0d0723a1402347f3ea/B in TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:49,859 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0014c64eb4fbb0d0723a1402347f3ea/A in TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:49,859 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/21c08b533bb4480986616cb2e1b1e4bd, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/fd74490d45f741e580d9274239f7c662, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/7911180183c549da912c85bce6680c6d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/a890be84de38486eb3231870dc26e74e] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp, totalSize=48.3 K 2024-11-15T22:13:49,859 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/b4c4a66be45241aaa87a972b0f943f31, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/31b68427263a4a9598e046a3a9753a7e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/69f3ea57120f42c5a1160359d2ea702c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/9819d07e08a74d889a5abc1389029d65] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp, totalSize=53.0 K 2024-11-15T22:13:49,859 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4c4a66be45241aaa87a972b0f943f31, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731708825900 2024-11-15T22:13:49,859 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 21c08b533bb4480986616cb2e1b1e4bd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731708825900 2024-11-15T22:13:49,859 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31b68427263a4a9598e046a3a9753a7e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731708825936 2024-11-15T22:13:49,859 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting fd74490d45f741e580d9274239f7c662, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731708825936 2024-11-15T22:13:49,860 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 7911180183c549da912c85bce6680c6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731708827126 2024-11-15T22:13:49,860 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69f3ea57120f42c5a1160359d2ea702c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731708827126 2024-11-15T22:13:49,860 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9819d07e08a74d889a5abc1389029d65, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731708828337 2024-11-15T22:13:49,860 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting a890be84de38486eb3231870dc26e74e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731708828337 2024-11-15T22:13:49,868 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0014c64eb4fbb0d0723a1402347f3ea#A#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:49,868 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/3dc69142102b45cba4fb17ccd89ad3eb is 50, key is test_row_0/A:col10/1731708828989/Put/seqid=0 2024-11-15T22:13:49,873 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0014c64eb4fbb0d0723a1402347f3ea#B#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:49,873 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/740dee262bdd438d9f7ff33cdf64b0d9 is 50, key is test_row_0/B:col10/1731708828989/Put/seqid=0 2024-11-15T22:13:49,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742361_1537 (size=12983) 2024-11-15T22:13:49,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742362_1538 (size=12983) 2024-11-15T22:13:49,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-15T22:13:49,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:49,907 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing d0014c64eb4fbb0d0723a1402347f3ea 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T22:13:49,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/19ff88e4fd0c474fb08e6757f6adc512 is 50, key is test_row_0/A:col10/1731708829024/Put/seqid=0 2024-11-15T22:13:49,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742363_1539 (size=12301) 2024-11-15T22:13:50,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-15T22:13:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. as already flushing 2024-11-15T22:13:50,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:50,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708890238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:50,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:50,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708890240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:50,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708890241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:50,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:50,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708890254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:50,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:50,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:50,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708890255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:50,286 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/3dc69142102b45cba4fb17ccd89ad3eb as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/3dc69142102b45cba4fb17ccd89ad3eb 2024-11-15T22:13:50,288 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/740dee262bdd438d9f7ff33cdf64b0d9 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/740dee262bdd438d9f7ff33cdf64b0d9 2024-11-15T22:13:50,291 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0014c64eb4fbb0d0723a1402347f3ea/A of d0014c64eb4fbb0d0723a1402347f3ea into 3dc69142102b45cba4fb17ccd89ad3eb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:50,291 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:50,291 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., storeName=d0014c64eb4fbb0d0723a1402347f3ea/A, priority=12, startTime=1731708829855; duration=0sec 2024-11-15T22:13:50,292 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:50,292 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0014c64eb4fbb0d0723a1402347f3ea:A 2024-11-15T22:13:50,292 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:13:50,293 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0014c64eb4fbb0d0723a1402347f3ea/B of d0014c64eb4fbb0d0723a1402347f3ea into 740dee262bdd438d9f7ff33cdf64b0d9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:50,293 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:50,293 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., storeName=d0014c64eb4fbb0d0723a1402347f3ea/B, priority=12, startTime=1731708829855; duration=0sec 2024-11-15T22:13:50,293 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:50,293 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0014c64eb4fbb0d0723a1402347f3ea:B 2024-11-15T22:13:50,293 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:13:50,294 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): d0014c64eb4fbb0d0723a1402347f3ea/C is initiating minor compaction (all files) 2024-11-15T22:13:50,294 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0014c64eb4fbb0d0723a1402347f3ea/C in TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:50,294 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/f2ac0186712848c892f0c4f69cd64320, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/864cc05cb1c34b51bb0dd0d0fb229dc7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/d9dfc1e666674e67a2fabca2a5eb8585, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/79a36ab418414fa093d33d3d87162341] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp, totalSize=48.3 K 2024-11-15T22:13:50,294 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2ac0186712848c892f0c4f69cd64320, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731708825900 2024-11-15T22:13:50,294 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 864cc05cb1c34b51bb0dd0d0fb229dc7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731708825936 2024-11-15T22:13:50,295 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9dfc1e666674e67a2fabca2a5eb8585, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731708827126 2024-11-15T22:13:50,295 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79a36ab418414fa093d33d3d87162341, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731708828337 2024-11-15T22:13:50,303 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0014c64eb4fbb0d0723a1402347f3ea#C#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:50,304 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/afbad5cad6454f73a8e8c3d1e1abbcf5 is 50, key is test_row_0/C:col10/1731708828989/Put/seqid=0 2024-11-15T22:13:50,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742364_1540 (size=12983) 2024-11-15T22:13:50,317 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/19ff88e4fd0c474fb08e6757f6adc512 2024-11-15T22:13:50,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/334386c7263f4aa7a9f2e4b79be1ceaa is 50, key is test_row_0/B:col10/1731708829024/Put/seqid=0 2024-11-15T22:13:50,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742365_1541 (size=12301) 2024-11-15T22:13:50,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:50,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708890366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:50,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:50,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708890366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:50,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708890366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:50,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:50,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708890377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:50,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:50,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708890377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:50,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:50,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:50,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:50,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708890576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:50,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708890576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:50,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:50,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708890577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:50,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708890587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708890587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:50,711 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/afbad5cad6454f73a8e8c3d1e1abbcf5 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/afbad5cad6454f73a8e8c3d1e1abbcf5 2024-11-15T22:13:50,716 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0014c64eb4fbb0d0723a1402347f3ea/C of d0014c64eb4fbb0d0723a1402347f3ea into afbad5cad6454f73a8e8c3d1e1abbcf5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:50,716 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:50,716 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., storeName=d0014c64eb4fbb0d0723a1402347f3ea/C, priority=12, startTime=1731708829855; duration=0sec 2024-11-15T22:13:50,716 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:50,716 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0014c64eb4fbb0d0723a1402347f3ea:C 2024-11-15T22:13:50,727 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/334386c7263f4aa7a9f2e4b79be1ceaa 2024-11-15T22:13:50,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/6206f8bbfa064a3b9f0a5a8a42815a00 is 50, key is test_row_0/C:col10/1731708829024/Put/seqid=0 2024-11-15T22:13:50,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742366_1542 (size=12301) 2024-11-15T22:13:50,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708890896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:50,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708890896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:50,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708890896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:50,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708890907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:50,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:50,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708890907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:50,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:50,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:51,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-15T22:13:51,138 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/6206f8bbfa064a3b9f0a5a8a42815a00 2024-11-15T22:13:51,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/19ff88e4fd0c474fb08e6757f6adc512 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/19ff88e4fd0c474fb08e6757f6adc512 2024-11-15T22:13:51,145 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/19ff88e4fd0c474fb08e6757f6adc512, entries=150, sequenceid=303, filesize=12.0 K 2024-11-15T22:13:51,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/334386c7263f4aa7a9f2e4b79be1ceaa as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/334386c7263f4aa7a9f2e4b79be1ceaa 2024-11-15T22:13:51,148 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/334386c7263f4aa7a9f2e4b79be1ceaa, entries=150, sequenceid=303, filesize=12.0 K 2024-11-15T22:13:51,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/6206f8bbfa064a3b9f0a5a8a42815a00 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/6206f8bbfa064a3b9f0a5a8a42815a00 2024-11-15T22:13:51,155 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/6206f8bbfa064a3b9f0a5a8a42815a00, entries=150, sequenceid=303, filesize=12.0 K 2024-11-15T22:13:51,156 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d0014c64eb4fbb0d0723a1402347f3ea in 1249ms, sequenceid=303, compaction requested=false 2024-11-15T22:13:51,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:51,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:51,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-15T22:13:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-15T22:13:51,159 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-15T22:13:51,159 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1680 sec 2024-11-15T22:13:51,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 2.1730 sec 2024-11-15T22:13:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:51,413 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:51,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0014c64eb4fbb0d0723a1402347f3ea 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T22:13:51,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/3d0212d62df146f0ae1d44d4b5d1c895 is 50, key is test_row_0/A:col10/1731708831412/Put/seqid=0 2024-11-15T22:13:51,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742367_1543 (size=14741) 2024-11-15T22:13:51,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708891456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:51,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:51,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708891470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:51,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:51,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708891471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:51,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708891472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:51,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708891474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:51,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708891577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:51,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708891587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:51,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:51,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708891587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:51,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:51,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708891587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:51,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708891596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:51,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708891797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:51,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708891797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:51,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708891797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:51,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708891806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:51,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:51,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708891806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:51,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:51,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/3d0212d62df146f0ae1d44d4b5d1c895 2024-11-15T22:13:51,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/cc963436e9b04b4f92ac3803927f2b5e is 50, key is test_row_0/B:col10/1731708831412/Put/seqid=0 2024-11-15T22:13:51,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742368_1544 (size=12301) 2024-11-15T22:13:51,912 INFO [Thread-2136 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2136. 2024-11-15T22:13:51,912 DEBUG [Thread-2136 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:13:51,912 DEBUG [Thread-2136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:51,912 DEBUG [Thread-2136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:51,912 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:51,916 INFO [Thread-2142 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2142. 2024-11-15T22:13:51,917 DEBUG [Thread-2142 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:13:51,917 DEBUG [Thread-2142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:51,917 DEBUG [Thread-2142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:51,917 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:51,917 INFO [Thread-2140 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2140. 2024-11-15T22:13:51,917 DEBUG [Thread-2140 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:13:51,917 DEBUG [Thread-2140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:51,917 DEBUG [Thread-2140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:51,917 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:51,922 INFO [Thread-2144 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2144. 2024-11-15T22:13:51,922 DEBUG [Thread-2144 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:13:51,922 DEBUG [Thread-2144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:51,922 DEBUG [Thread-2144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:51,922 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:51,924 INFO [Thread-2138 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2138. 2024-11-15T22:13:51,924 DEBUG [Thread-2138 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:13:51,924 DEBUG [Thread-2138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:51,924 DEBUG [Thread-2138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:51,924 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:52,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:52,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708892106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:52,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:52,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:52,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:52,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:52,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708892117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708892117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:52,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:52,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708892126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708892126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:52,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/cc963436e9b04b4f92ac3803927f2b5e 2024-11-15T22:13:52,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/8e871f87d1754669975ad44a40816f1e is 50, key is test_row_0/C:col10/1731708831412/Put/seqid=0 2024-11-15T22:13:52,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742369_1545 (size=12301) 2024-11-15T22:13:52,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47712 deadline: 1731708892616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:52,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:52,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:52,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:52,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:52,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47764 deadline: 1731708892626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47698 deadline: 1731708892627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:52,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1731708892637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:52,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:13:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47758 deadline: 1731708892637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:52,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:52,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:52,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:52,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 , the old value is region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:52,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:13:52,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=2 because the exception is null or not the one we care about 2024-11-15T22:13:52,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/8e871f87d1754669975ad44a40816f1e 2024-11-15T22:13:52,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/3d0212d62df146f0ae1d44d4b5d1c895 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/3d0212d62df146f0ae1d44d4b5d1c895 2024-11-15T22:13:52,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/3d0212d62df146f0ae1d44d4b5d1c895, entries=200, sequenceid=327, filesize=14.4 K 2024-11-15T22:13:52,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/cc963436e9b04b4f92ac3803927f2b5e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/cc963436e9b04b4f92ac3803927f2b5e 2024-11-15T22:13:52,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/cc963436e9b04b4f92ac3803927f2b5e, entries=150, sequenceid=327, filesize=12.0 K 2024-11-15T22:13:52,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/8e871f87d1754669975ad44a40816f1e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/8e871f87d1754669975ad44a40816f1e 2024-11-15T22:13:52,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/8e871f87d1754669975ad44a40816f1e, entries=150, sequenceid=327, filesize=12.0 K 2024-11-15T22:13:52,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for d0014c64eb4fbb0d0723a1402347f3ea in 1240ms, sequenceid=327, compaction requested=true 2024-11-15T22:13:52,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:52,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0014c64eb4fbb0d0723a1402347f3ea:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:13:52,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:52,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0014c64eb4fbb0d0723a1402347f3ea:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:13:52,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:52,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0014c64eb4fbb0d0723a1402347f3ea:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:13:52,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:52,653 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:52,653 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:52,654 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:52,654 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:52,654 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): d0014c64eb4fbb0d0723a1402347f3ea/B is initiating minor compaction (all files) 2024-11-15T22:13:52,654 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): d0014c64eb4fbb0d0723a1402347f3ea/A is initiating minor compaction (all files) 2024-11-15T22:13:52,654 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0014c64eb4fbb0d0723a1402347f3ea/B in TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:52,654 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0014c64eb4fbb0d0723a1402347f3ea/A in TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:52,654 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/3dc69142102b45cba4fb17ccd89ad3eb, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/19ff88e4fd0c474fb08e6757f6adc512, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/3d0212d62df146f0ae1d44d4b5d1c895] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp, totalSize=39.1 K 2024-11-15T22:13:52,654 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/740dee262bdd438d9f7ff33cdf64b0d9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/334386c7263f4aa7a9f2e4b79be1ceaa, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/cc963436e9b04b4f92ac3803927f2b5e] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp, totalSize=36.7 K 2024-11-15T22:13:52,655 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dc69142102b45cba4fb17ccd89ad3eb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731708828337 2024-11-15T22:13:52,655 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 740dee262bdd438d9f7ff33cdf64b0d9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731708828337 2024-11-15T22:13:52,655 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 334386c7263f4aa7a9f2e4b79be1ceaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731708829020 2024-11-15T22:13:52,655 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19ff88e4fd0c474fb08e6757f6adc512, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731708829020 2024-11-15T22:13:52,655 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting cc963436e9b04b4f92ac3803927f2b5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731708830238 2024-11-15T22:13:52,655 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d0212d62df146f0ae1d44d4b5d1c895, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731708830238 2024-11-15T22:13:52,661 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0014c64eb4fbb0d0723a1402347f3ea#A#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:52,661 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0014c64eb4fbb0d0723a1402347f3ea#B#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:52,661 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/cc468234334a433ab06094a8fe86bc8a is 50, key is test_row_0/A:col10/1731708831412/Put/seqid=0 2024-11-15T22:13:52,661 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/cceda6a3ba5e4de7b06da4a9386d4e31 is 50, key is test_row_0/B:col10/1731708831412/Put/seqid=0 2024-11-15T22:13:52,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742370_1546 (size=13085) 2024-11-15T22:13:52,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742371_1547 (size=13085) 2024-11-15T22:13:53,068 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/cceda6a3ba5e4de7b06da4a9386d4e31 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/cceda6a3ba5e4de7b06da4a9386d4e31 2024-11-15T22:13:53,068 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/cc468234334a433ab06094a8fe86bc8a as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/cc468234334a433ab06094a8fe86bc8a 2024-11-15T22:13:53,072 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0014c64eb4fbb0d0723a1402347f3ea/A of d0014c64eb4fbb0d0723a1402347f3ea into cc468234334a433ab06094a8fe86bc8a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:53,072 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0014c64eb4fbb0d0723a1402347f3ea/B of d0014c64eb4fbb0d0723a1402347f3ea into cceda6a3ba5e4de7b06da4a9386d4e31(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:53,072 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:53,072 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:53,072 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., storeName=d0014c64eb4fbb0d0723a1402347f3ea/B, priority=13, startTime=1731708832653; duration=0sec 2024-11-15T22:13:53,072 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., storeName=d0014c64eb4fbb0d0723a1402347f3ea/A, priority=13, startTime=1731708832653; duration=0sec 2024-11-15T22:13:53,072 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:53,072 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0014c64eb4fbb0d0723a1402347f3ea:A 2024-11-15T22:13:53,072 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:13:53,072 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0014c64eb4fbb0d0723a1402347f3ea:B 2024-11-15T22:13:53,072 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:13:53,073 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:13:53,073 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): d0014c64eb4fbb0d0723a1402347f3ea/C is initiating minor compaction (all files) 2024-11-15T22:13:53,073 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0014c64eb4fbb0d0723a1402347f3ea/C in TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:53,073 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/afbad5cad6454f73a8e8c3d1e1abbcf5, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/6206f8bbfa064a3b9f0a5a8a42815a00, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/8e871f87d1754669975ad44a40816f1e] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp, totalSize=36.7 K 2024-11-15T22:13:53,073 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting afbad5cad6454f73a8e8c3d1e1abbcf5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731708828337 2024-11-15T22:13:53,074 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6206f8bbfa064a3b9f0a5a8a42815a00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731708829020 2024-11-15T22:13:53,074 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e871f87d1754669975ad44a40816f1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731708830238 2024-11-15T22:13:53,079 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0014c64eb4fbb0d0723a1402347f3ea#C#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:13:53,080 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/7f68835817b44ba5a2b78b7aeb6ceea1 is 50, key is test_row_0/C:col10/1731708831412/Put/seqid=0 2024-11-15T22:13:53,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742372_1548 (size=13085) 2024-11-15T22:13:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-15T22:13:53,157 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:53,487 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/7f68835817b44ba5a2b78b7aeb6ceea1 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/7f68835817b44ba5a2b78b7aeb6ceea1 2024-11-15T22:13:53,491 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0014c64eb4fbb0d0723a1402347f3ea/C of d0014c64eb4fbb0d0723a1402347f3ea into 7f68835817b44ba5a2b78b7aeb6ceea1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:13:53,491 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:53,491 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., storeName=d0014c64eb4fbb0d0723a1402347f3ea/C, priority=13, startTime=1731708832653; duration=0sec 2024-11-15T22:13:53,491 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:13:53,491 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0014c64eb4fbb0d0723a1402347f3ea:C 2024-11-15T22:13:53,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:53,628 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:13:53,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0014c64eb4fbb0d0723a1402347f3ea 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T22:13:53,628 INFO [Thread-2129 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2129. 2024-11-15T22:13:53,628 DEBUG [Thread-2129 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:13:53,628 DEBUG [Thread-2129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:53,628 DEBUG [Thread-2129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:53,628 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:53,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/63cd8eb3f7b14bd3872b69b17de51c5e is 50, key is test_row_0/A:col10/1731708833627/Put/seqid=0 2024-11-15T22:13:53,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742373_1549 (size=12301) 2024-11-15T22:13:53,638 INFO [Thread-2127 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2127. 2024-11-15T22:13:53,638 DEBUG [Thread-2127 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:13:53,638 INFO [Thread-2125 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2125. 2024-11-15T22:13:53,638 DEBUG [Thread-2127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:53,638 DEBUG [Thread-2125 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:13:53,638 DEBUG [Thread-2125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:53,638 DEBUG [Thread-2127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:53,638 DEBUG [Thread-2125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:53,638 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:53,638 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:53,647 INFO [Thread-2133 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2133. 2024-11-15T22:13:53,647 INFO [Thread-2131 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2131. 2024-11-15T22:13:53,647 DEBUG [Thread-2133 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:13:53,647 DEBUG [Thread-2131 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:13:53,647 DEBUG [Thread-2133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:53,647 DEBUG [Thread-2131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:53,647 DEBUG [Thread-2133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:53,647 DEBUG [Thread-2131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:53,647 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:53,647 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:53,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T22:13:53,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-15T22:13:53,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-15T22:13:53,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-15T22:13:53,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-15T22:13:53,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-15T22:13:53,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T22:13:53,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T22:13:53,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1183 2024-11-15T22:13:53,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3549 rows 2024-11-15T22:13:53,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1200 2024-11-15T22:13:53,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3598 rows 2024-11-15T22:13:53,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1193 2024-11-15T22:13:53,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3577 rows 2024-11-15T22:13:53,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1199 2024-11-15T22:13:53,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3597 rows 2024-11-15T22:13:53,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1193 2024-11-15T22:13:53,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3576 rows 2024-11-15T22:13:53,648 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T22:13:53,648 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:13:53,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:53,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:13:53,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:13:53,649 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T22:13:53,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T22:13:53,651 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708833651"}]},"ts":"1731708833651"} 2024-11-15T22:13:53,652 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T22:13:53,652 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T22:13:53,653 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T22:13:53,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0014c64eb4fbb0d0723a1402347f3ea, UNASSIGN}] 2024-11-15T22:13:53,655 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0014c64eb4fbb0d0723a1402347f3ea, UNASSIGN 2024-11-15T22:13:53,655 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=d0014c64eb4fbb0d0723a1402347f3ea, regionState=CLOSING, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:53,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0014c64eb4fbb0d0723a1402347f3ea, UNASSIGN because future has completed 2024-11-15T22:13:53,656 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T22:13:53,657 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516}] 2024-11-15T22:13:53,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T22:13:53,808 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:53,809 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T22:13:53,809 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing d0014c64eb4fbb0d0723a1402347f3ea, disabling compactions & flushes 2024-11-15T22:13:53,809 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T22:13:54,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/63cd8eb3f7b14bd3872b69b17de51c5e 2024-11-15T22:13:54,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/950678459c1c42d5886c25a681158d96 is 50, key is test_row_0/B:col10/1731708833627/Put/seqid=0 2024-11-15T22:13:54,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742374_1550 (size=12301) 2024-11-15T22:13:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T22:13:54,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/950678459c1c42d5886c25a681158d96 2024-11-15T22:13:54,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/56e16e24a74a423c876c5e2b3b0a25d9 is 50, key is test_row_0/C:col10/1731708833627/Put/seqid=0 2024-11-15T22:13:54,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742375_1551 (size=12301) 2024-11-15T22:13:54,574 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T22:13:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T22:13:54,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/56e16e24a74a423c876c5e2b3b0a25d9 2024-11-15T22:13:54,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/63cd8eb3f7b14bd3872b69b17de51c5e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/63cd8eb3f7b14bd3872b69b17de51c5e 2024-11-15T22:13:54,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/63cd8eb3f7b14bd3872b69b17de51c5e, entries=150, sequenceid=346, filesize=12.0 K 2024-11-15T22:13:54,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/950678459c1c42d5886c25a681158d96 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/950678459c1c42d5886c25a681158d96 2024-11-15T22:13:54,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/950678459c1c42d5886c25a681158d96, entries=150, sequenceid=346, filesize=12.0 K 2024-11-15T22:13:54,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/56e16e24a74a423c876c5e2b3b0a25d9 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/56e16e24a74a423c876c5e2b3b0a25d9 2024-11-15T22:13:54,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/56e16e24a74a423c876c5e2b3b0a25d9, entries=150, sequenceid=346, filesize=12.0 K 2024-11-15T22:13:54,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=26.84 KB/27480 for d0014c64eb4fbb0d0723a1402347f3ea in 1240ms, sequenceid=346, compaction requested=false 2024-11-15T22:13:54,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0014c64eb4fbb0d0723a1402347f3ea: 2024-11-15T22:13:54,868 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:54,868 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:54,868 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. after waiting 0 ms 2024-11-15T22:13:54,868 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:54,868 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(2902): Flushing d0014c64eb4fbb0d0723a1402347f3ea 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-15T22:13:54,871 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/12ee2bfaee0d427d875da76cd0bdd892 is 50, key is test_row_0/A:col10/1731708833646/Put/seqid=0 2024-11-15T22:13:54,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742376_1552 (size=12301) 2024-11-15T22:13:55,275 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/12ee2bfaee0d427d875da76cd0bdd892 2024-11-15T22:13:55,280 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/cb969048fd13435c8485bb51bdf3f1a5 is 50, key is test_row_0/B:col10/1731708833646/Put/seqid=0 2024-11-15T22:13:55,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742377_1553 (size=12301) 2024-11-15T22:13:55,684 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/cb969048fd13435c8485bb51bdf3f1a5 2024-11-15T22:13:55,690 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/ecdadbd734e04953808ab208fdb5020c is 50, key is test_row_0/C:col10/1731708833646/Put/seqid=0 2024-11-15T22:13:55,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742378_1554 (size=12301) 2024-11-15T22:13:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T22:13:56,093 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/ecdadbd734e04953808ab208fdb5020c 2024-11-15T22:13:56,097 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/A/12ee2bfaee0d427d875da76cd0bdd892 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/12ee2bfaee0d427d875da76cd0bdd892 2024-11-15T22:13:56,100 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/12ee2bfaee0d427d875da76cd0bdd892, entries=150, sequenceid=353, filesize=12.0 K 2024-11-15T22:13:56,100 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/B/cb969048fd13435c8485bb51bdf3f1a5 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/cb969048fd13435c8485bb51bdf3f1a5 2024-11-15T22:13:56,103 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/cb969048fd13435c8485bb51bdf3f1a5, entries=150, sequenceid=353, filesize=12.0 K 2024-11-15T22:13:56,104 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/.tmp/C/ecdadbd734e04953808ab208fdb5020c as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/ecdadbd734e04953808ab208fdb5020c 2024-11-15T22:13:56,106 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/ecdadbd734e04953808ab208fdb5020c, entries=150, sequenceid=353, filesize=12.0 K 2024-11-15T22:13:56,107 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for d0014c64eb4fbb0d0723a1402347f3ea in 1239ms, sequenceid=353, compaction requested=true 2024-11-15T22:13:56,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/4b0bbbfe01f944d8815f761c4f4b6f37, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/85148d3424e04bd69960b414511ecbc7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/a0d3df1f399b4281a6c7f63aa53ba7a1, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/a569be8b431146859fd8ab10b089dd42, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/12f9191635e344779f1f2fcccff39cc5, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/68d43b6d8cd34d0cb1df3e7de5647451, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/9531bf7f19c946eb866659aa5b4c1a62, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/ca3efac236794637a545fd66e1809902, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/868e0b2476d54692a77ce6ebd2c5dea7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/7bf138930aa64f178502fe055887f0cd, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/e767ea25ec4d416a8863b8713c31cadd, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/455adcf24b144fe7aca7a7c2dd2d5f7c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/0ca667ace35b46cca83e5b590fb2a261, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/f1754f21512e4c47ae626b63483a6ac2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/6fc77560dfe24a62909181e657ba5b2c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/597a55f1d0654fe28fb6c6ba610b2cb0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/b4c4a66be45241aaa87a972b0f943f31, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/31b68427263a4a9598e046a3a9753a7e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/69f3ea57120f42c5a1160359d2ea702c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/9819d07e08a74d889a5abc1389029d65, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/3dc69142102b45cba4fb17ccd89ad3eb, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/19ff88e4fd0c474fb08e6757f6adc512, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/3d0212d62df146f0ae1d44d4b5d1c895] to archive 2024-11-15T22:13:56,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:13:56,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/4b0bbbfe01f944d8815f761c4f4b6f37 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/4b0bbbfe01f944d8815f761c4f4b6f37 2024-11-15T22:13:56,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/85148d3424e04bd69960b414511ecbc7 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/85148d3424e04bd69960b414511ecbc7 2024-11-15T22:13:56,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/a0d3df1f399b4281a6c7f63aa53ba7a1 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/a0d3df1f399b4281a6c7f63aa53ba7a1 2024-11-15T22:13:56,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/a569be8b431146859fd8ab10b089dd42 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/a569be8b431146859fd8ab10b089dd42 2024-11-15T22:13:56,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/12f9191635e344779f1f2fcccff39cc5 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/12f9191635e344779f1f2fcccff39cc5 2024-11-15T22:13:56,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/68d43b6d8cd34d0cb1df3e7de5647451 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/68d43b6d8cd34d0cb1df3e7de5647451 2024-11-15T22:13:56,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/9531bf7f19c946eb866659aa5b4c1a62 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/9531bf7f19c946eb866659aa5b4c1a62 2024-11-15T22:13:56,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/ca3efac236794637a545fd66e1809902 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/ca3efac236794637a545fd66e1809902 2024-11-15T22:13:56,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/868e0b2476d54692a77ce6ebd2c5dea7 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/868e0b2476d54692a77ce6ebd2c5dea7 2024-11-15T22:13:56,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/7bf138930aa64f178502fe055887f0cd to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/7bf138930aa64f178502fe055887f0cd 2024-11-15T22:13:56,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/e767ea25ec4d416a8863b8713c31cadd to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/e767ea25ec4d416a8863b8713c31cadd 2024-11-15T22:13:56,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/455adcf24b144fe7aca7a7c2dd2d5f7c to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/455adcf24b144fe7aca7a7c2dd2d5f7c 2024-11-15T22:13:56,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/0ca667ace35b46cca83e5b590fb2a261 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/0ca667ace35b46cca83e5b590fb2a261 2024-11-15T22:13:56,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/f1754f21512e4c47ae626b63483a6ac2 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/f1754f21512e4c47ae626b63483a6ac2 2024-11-15T22:13:56,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/6fc77560dfe24a62909181e657ba5b2c to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/6fc77560dfe24a62909181e657ba5b2c 2024-11-15T22:13:56,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/597a55f1d0654fe28fb6c6ba610b2cb0 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/597a55f1d0654fe28fb6c6ba610b2cb0 2024-11-15T22:13:56,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/b4c4a66be45241aaa87a972b0f943f31 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/b4c4a66be45241aaa87a972b0f943f31 2024-11-15T22:13:56,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/31b68427263a4a9598e046a3a9753a7e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/31b68427263a4a9598e046a3a9753a7e 2024-11-15T22:13:56,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/69f3ea57120f42c5a1160359d2ea702c to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/69f3ea57120f42c5a1160359d2ea702c 2024-11-15T22:13:56,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/9819d07e08a74d889a5abc1389029d65 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/9819d07e08a74d889a5abc1389029d65 2024-11-15T22:13:56,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/3dc69142102b45cba4fb17ccd89ad3eb to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/3dc69142102b45cba4fb17ccd89ad3eb 2024-11-15T22:13:56,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/19ff88e4fd0c474fb08e6757f6adc512 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/19ff88e4fd0c474fb08e6757f6adc512 2024-11-15T22:13:56,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/3d0212d62df146f0ae1d44d4b5d1c895 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/3d0212d62df146f0ae1d44d4b5d1c895 2024-11-15T22:13:56,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/94cd9872ef5f429c8279e0fd6308cb8f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ffbb4bd2cadd433b932da698f1081af1, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ae887d1f70024a19b5418865bed95eb0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ea424e6f9ef0484b8628a74120740f43, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/fd13285cdfd54bb1b823d986889e18c8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/c3dc5a8d7c1a4f5a94cb6d03a278006c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/3592a53a894243bb820bd0ac3e10e16c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/e507f49366f24e83a6781549bb4d4ea0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/b3a7ba64c91e4f1eb80896fa59f1fa1b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/625c18e8c33e41e49a4a77ac7f798602, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/3213d7bb121b4deb848f085d0fd267ad, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/126e07ae578d4392b22a9020b8c4b03d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ab6cc2e5f44c4f7d9552d9a6a2df6484, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/b8324fa4627d495f966b581315d7ae58, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/419da4359a6b426d8228f8d41af33bea, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/21c08b533bb4480986616cb2e1b1e4bd, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/93c4d3f7ac1c4b538b318e8e11c12a8c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/fd74490d45f741e580d9274239f7c662, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/7911180183c549da912c85bce6680c6d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/740dee262bdd438d9f7ff33cdf64b0d9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/a890be84de38486eb3231870dc26e74e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/334386c7263f4aa7a9f2e4b79be1ceaa, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/cc963436e9b04b4f92ac3803927f2b5e] to archive 2024-11-15T22:13:56,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:13:56,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/94cd9872ef5f429c8279e0fd6308cb8f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/94cd9872ef5f429c8279e0fd6308cb8f 2024-11-15T22:13:56,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ffbb4bd2cadd433b932da698f1081af1 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ffbb4bd2cadd433b932da698f1081af1 2024-11-15T22:13:56,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ae887d1f70024a19b5418865bed95eb0 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ae887d1f70024a19b5418865bed95eb0 2024-11-15T22:13:56,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ea424e6f9ef0484b8628a74120740f43 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ea424e6f9ef0484b8628a74120740f43 2024-11-15T22:13:56,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/fd13285cdfd54bb1b823d986889e18c8 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/fd13285cdfd54bb1b823d986889e18c8 2024-11-15T22:13:56,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/c3dc5a8d7c1a4f5a94cb6d03a278006c to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/c3dc5a8d7c1a4f5a94cb6d03a278006c 2024-11-15T22:13:56,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/3592a53a894243bb820bd0ac3e10e16c to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/3592a53a894243bb820bd0ac3e10e16c 2024-11-15T22:13:56,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/e507f49366f24e83a6781549bb4d4ea0 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/e507f49366f24e83a6781549bb4d4ea0 2024-11-15T22:13:56,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/b3a7ba64c91e4f1eb80896fa59f1fa1b to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/b3a7ba64c91e4f1eb80896fa59f1fa1b 2024-11-15T22:13:56,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/625c18e8c33e41e49a4a77ac7f798602 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/625c18e8c33e41e49a4a77ac7f798602 2024-11-15T22:13:56,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/3213d7bb121b4deb848f085d0fd267ad to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/3213d7bb121b4deb848f085d0fd267ad 2024-11-15T22:13:56,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/126e07ae578d4392b22a9020b8c4b03d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/126e07ae578d4392b22a9020b8c4b03d 2024-11-15T22:13:56,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ab6cc2e5f44c4f7d9552d9a6a2df6484 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/ab6cc2e5f44c4f7d9552d9a6a2df6484 2024-11-15T22:13:56,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/b8324fa4627d495f966b581315d7ae58 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/b8324fa4627d495f966b581315d7ae58 2024-11-15T22:13:56,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/419da4359a6b426d8228f8d41af33bea to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/419da4359a6b426d8228f8d41af33bea 2024-11-15T22:13:56,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/21c08b533bb4480986616cb2e1b1e4bd to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/21c08b533bb4480986616cb2e1b1e4bd 2024-11-15T22:13:56,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/93c4d3f7ac1c4b538b318e8e11c12a8c to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/93c4d3f7ac1c4b538b318e8e11c12a8c 2024-11-15T22:13:56,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/fd74490d45f741e580d9274239f7c662 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/fd74490d45f741e580d9274239f7c662 2024-11-15T22:13:56,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/7911180183c549da912c85bce6680c6d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/7911180183c549da912c85bce6680c6d 2024-11-15T22:13:56,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/740dee262bdd438d9f7ff33cdf64b0d9 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/740dee262bdd438d9f7ff33cdf64b0d9 2024-11-15T22:13:56,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/a890be84de38486eb3231870dc26e74e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/a890be84de38486eb3231870dc26e74e 2024-11-15T22:13:56,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/334386c7263f4aa7a9f2e4b79be1ceaa to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/334386c7263f4aa7a9f2e4b79be1ceaa 2024-11-15T22:13:56,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/cc963436e9b04b4f92ac3803927f2b5e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/cc963436e9b04b4f92ac3803927f2b5e 2024-11-15T22:13:56,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/a535f1294611440cba285fa234080780, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/5cafb610329b4aa0b400033da1a45982, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/528b65322227404da662bf4f28b8812d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/0f8217404bfb4c3ead03e146f9fb3410, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/ff3547ea1ef14f56b79e51c941f604ee, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/5d2b55fc2569445f88152caccba6c176, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/e49f1eea1bbf471c8ab40c0ba56886d0, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/8251c79ee6944251af3dbffde5afcfb7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/aa889b1183264d2eb3467bbcce564597, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/c2ed1f7cb40344cab2cf72b1d6ae66c7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/ac1a09a4b67a4ce7b6b588474be3c6af, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/7c844ed36a314b15b391fbe11ba5c2e5, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/7c881285956b4b378f2bc9039041f2d6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/48d96f28751840778853f8f964a9b43a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/c0449cf237d34b578a1163480ec61e20, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/f2ac0186712848c892f0c4f69cd64320, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/7b31880cb7d8461bb2c8c312a96564c6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/864cc05cb1c34b51bb0dd0d0fb229dc7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/d9dfc1e666674e67a2fabca2a5eb8585, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/afbad5cad6454f73a8e8c3d1e1abbcf5, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/79a36ab418414fa093d33d3d87162341, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/6206f8bbfa064a3b9f0a5a8a42815a00, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/8e871f87d1754669975ad44a40816f1e] to archive 2024-11-15T22:13:56,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:13:56,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/a535f1294611440cba285fa234080780 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/a535f1294611440cba285fa234080780 2024-11-15T22:13:56,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/5cafb610329b4aa0b400033da1a45982 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/5cafb610329b4aa0b400033da1a45982 2024-11-15T22:13:56,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/528b65322227404da662bf4f28b8812d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/528b65322227404da662bf4f28b8812d 2024-11-15T22:13:56,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/0f8217404bfb4c3ead03e146f9fb3410 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/0f8217404bfb4c3ead03e146f9fb3410 2024-11-15T22:13:56,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/ff3547ea1ef14f56b79e51c941f604ee to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/ff3547ea1ef14f56b79e51c941f604ee 2024-11-15T22:13:56,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/5d2b55fc2569445f88152caccba6c176 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/5d2b55fc2569445f88152caccba6c176 2024-11-15T22:13:56,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/e49f1eea1bbf471c8ab40c0ba56886d0 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/e49f1eea1bbf471c8ab40c0ba56886d0 2024-11-15T22:13:56,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/8251c79ee6944251af3dbffde5afcfb7 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/8251c79ee6944251af3dbffde5afcfb7 2024-11-15T22:13:56,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/aa889b1183264d2eb3467bbcce564597 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/aa889b1183264d2eb3467bbcce564597 2024-11-15T22:13:56,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/c2ed1f7cb40344cab2cf72b1d6ae66c7 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/c2ed1f7cb40344cab2cf72b1d6ae66c7 2024-11-15T22:13:56,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/ac1a09a4b67a4ce7b6b588474be3c6af to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/ac1a09a4b67a4ce7b6b588474be3c6af 2024-11-15T22:13:56,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/7c844ed36a314b15b391fbe11ba5c2e5 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/7c844ed36a314b15b391fbe11ba5c2e5 2024-11-15T22:13:56,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/7c881285956b4b378f2bc9039041f2d6 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/7c881285956b4b378f2bc9039041f2d6 2024-11-15T22:13:56,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/48d96f28751840778853f8f964a9b43a to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/48d96f28751840778853f8f964a9b43a 2024-11-15T22:13:56,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/c0449cf237d34b578a1163480ec61e20 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/c0449cf237d34b578a1163480ec61e20 2024-11-15T22:13:56,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/f2ac0186712848c892f0c4f69cd64320 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/f2ac0186712848c892f0c4f69cd64320 2024-11-15T22:13:56,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/7b31880cb7d8461bb2c8c312a96564c6 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/7b31880cb7d8461bb2c8c312a96564c6 2024-11-15T22:13:56,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/864cc05cb1c34b51bb0dd0d0fb229dc7 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/864cc05cb1c34b51bb0dd0d0fb229dc7 2024-11-15T22:13:56,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/d9dfc1e666674e67a2fabca2a5eb8585 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/d9dfc1e666674e67a2fabca2a5eb8585 2024-11-15T22:13:56,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/afbad5cad6454f73a8e8c3d1e1abbcf5 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/afbad5cad6454f73a8e8c3d1e1abbcf5 2024-11-15T22:13:56,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/79a36ab418414fa093d33d3d87162341 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/79a36ab418414fa093d33d3d87162341 2024-11-15T22:13:56,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/6206f8bbfa064a3b9f0a5a8a42815a00 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/6206f8bbfa064a3b9f0a5a8a42815a00 2024-11-15T22:13:56,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/8e871f87d1754669975ad44a40816f1e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/8e871f87d1754669975ad44a40816f1e 2024-11-15T22:13:56,172 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/recovered.edits/356.seqid, newMaxSeqId=356, maxSeqId=1 2024-11-15T22:13:56,173 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. 2024-11-15T22:13:56,173 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for d0014c64eb4fbb0d0723a1402347f3ea: Waiting for close lock at 1731708833809Running coprocessor pre-close hooks at 1731708833809Disabling compacts and flushes for region at 1731708833809Disabling writes for close at 1731708834868 (+1059 ms)Obtaining lock to block concurrent updates at 1731708834868Preparing flush snapshotting stores in d0014c64eb4fbb0d0723a1402347f3ea at 1731708834868Finished memstore snapshotting TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731708834868Flushing stores of TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea. at 1731708834869 (+1 ms)Flushing d0014c64eb4fbb0d0723a1402347f3ea/A: creating writer at 1731708834869Flushing d0014c64eb4fbb0d0723a1402347f3ea/A: appending metadata at 1731708834871 (+2 ms)Flushing d0014c64eb4fbb0d0723a1402347f3ea/A: closing flushed file at 1731708834871Flushing d0014c64eb4fbb0d0723a1402347f3ea/B: creating writer at 1731708835278 (+407 ms)Flushing d0014c64eb4fbb0d0723a1402347f3ea/B: appending metadata at 1731708835280 (+2 ms)Flushing d0014c64eb4fbb0d0723a1402347f3ea/B: closing flushed file at 1731708835280Flushing d0014c64eb4fbb0d0723a1402347f3ea/C: creating writer at 1731708835687 (+407 ms)Flushing d0014c64eb4fbb0d0723a1402347f3ea/C: appending metadata at 1731708835689 (+2 ms)Flushing d0014c64eb4fbb0d0723a1402347f3ea/C: closing flushed file at 1731708835689Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24bf6b13: reopening flushed file at 1731708836096 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23e28fc4: reopening flushed file at 1731708836100 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5df266c3: reopening flushed file at 1731708836103 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for d0014c64eb4fbb0d0723a1402347f3ea in 1239ms, sequenceid=353, compaction requested=true at 1731708836107 (+4 ms)Writing region close event to WAL at 1731708836170 (+63 ms)Running coprocessor post-close hooks at 1731708836173 (+3 ms)Closed at 1731708836173 2024-11-15T22:13:56,175 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:56,175 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=d0014c64eb4fbb0d0723a1402347f3ea, regionState=CLOSED 2024-11-15T22:13:56,176 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 because future has completed 2024-11-15T22:13:56,178 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-15T22:13:56,178 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure d0014c64eb4fbb0d0723a1402347f3ea, server=ae3311ea5ccb,37917,1731708686516 in 2.5200 sec 2024-11-15T22:13:56,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-15T22:13:56,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0014c64eb4fbb0d0723a1402347f3ea, UNASSIGN in 2.5240 sec 2024-11-15T22:13:56,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-15T22:13:56,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5270 sec 2024-11-15T22:13:56,183 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708836183"}]},"ts":"1731708836183"} 2024-11-15T22:13:56,184 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T22:13:56,184 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T22:13:56,185 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5350 sec 2024-11-15T22:13:57,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T22:13:57,816 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:57,817 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T22:13:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:57,819 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-15T22:13:57,820 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=147, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:57,821 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:57,823 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A, FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B, FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C, FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/recovered.edits] 2024-11-15T22:13:57,825 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/12ee2bfaee0d427d875da76cd0bdd892 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/12ee2bfaee0d427d875da76cd0bdd892 2024-11-15T22:13:57,826 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/63cd8eb3f7b14bd3872b69b17de51c5e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/63cd8eb3f7b14bd3872b69b17de51c5e 2024-11-15T22:13:57,826 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/cc468234334a433ab06094a8fe86bc8a to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/A/cc468234334a433ab06094a8fe86bc8a 2024-11-15T22:13:57,828 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/950678459c1c42d5886c25a681158d96 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/950678459c1c42d5886c25a681158d96 2024-11-15T22:13:57,829 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/cb969048fd13435c8485bb51bdf3f1a5 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/cb969048fd13435c8485bb51bdf3f1a5 2024-11-15T22:13:57,830 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/cceda6a3ba5e4de7b06da4a9386d4e31 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/B/cceda6a3ba5e4de7b06da4a9386d4e31 2024-11-15T22:13:57,831 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/56e16e24a74a423c876c5e2b3b0a25d9 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/56e16e24a74a423c876c5e2b3b0a25d9 2024-11-15T22:13:57,832 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/7f68835817b44ba5a2b78b7aeb6ceea1 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/7f68835817b44ba5a2b78b7aeb6ceea1 2024-11-15T22:13:57,833 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/ecdadbd734e04953808ab208fdb5020c to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/C/ecdadbd734e04953808ab208fdb5020c 2024-11-15T22:13:57,835 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/recovered.edits/356.seqid to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea/recovered.edits/356.seqid 2024-11-15T22:13:57,835 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/d0014c64eb4fbb0d0723a1402347f3ea 2024-11-15T22:13:57,835 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T22:13:57,837 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=147, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:57,839 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T22:13:57,840 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T22:13:57,841 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=147, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:57,841 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T22:13:57,842 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731708837841"}]},"ts":"9223372036854775807"} 2024-11-15T22:13:57,843 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T22:13:57,843 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d0014c64eb4fbb0d0723a1402347f3ea, NAME => 'TestAcidGuarantees,,1731708809711.d0014c64eb4fbb0d0723a1402347f3ea.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T22:13:57,843 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T22:13:57,843 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731708837843"}]},"ts":"9223372036854775807"} 2024-11-15T22:13:57,845 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T22:13:57,845 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=147, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:57,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 28 msec 2024-11-15T22:13:57,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-15T22:13:57,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T22:13:57,926 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:13:57,935 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241 (was 242), OpenFileDescriptor=449 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=604 (was 710), ProcessCount=11 (was 11), AvailableMemoryMB=2228 (was 2642) 2024-11-15T22:13:57,944 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=604, ProcessCount=11, AvailableMemoryMB=2227 2024-11-15T22:13:57,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T22:13:57,945 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:13:57,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T22:13:57,947 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T22:13:57,947 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 148 2024-11-15T22:13:57,947 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:57,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T22:13:57,948 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T22:13:57,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742379_1555 (size=959) 2024-11-15T22:13:58,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T22:13:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T22:13:58,355 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31 2024-11-15T22:13:58,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742380_1556 (size=53) 2024-11-15T22:13:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T22:13:58,761 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:13:58,761 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 28230b65c0b592a41b759dc13b865232, disabling compactions & flushes 2024-11-15T22:13:58,761 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:13:58,761 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:13:58,761 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. after waiting 0 ms 2024-11-15T22:13:58,761 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:13:58,761 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:13:58,761 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 28230b65c0b592a41b759dc13b865232: Waiting for close lock at 1731708838761Disabling compacts and flushes for region at 1731708838761Disabling writes for close at 1731708838761Writing region close event to WAL at 1731708838761Closed at 1731708838761 2024-11-15T22:13:58,762 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T22:13:58,762 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731708838762"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708838762"}]},"ts":"1731708838762"} 2024-11-15T22:13:58,764 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T22:13:58,764 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T22:13:58,764 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708838764"}]},"ts":"1731708838764"} 2024-11-15T22:13:58,766 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T22:13:58,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=28230b65c0b592a41b759dc13b865232, ASSIGN}] 2024-11-15T22:13:58,767 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=28230b65c0b592a41b759dc13b865232, ASSIGN 2024-11-15T22:13:58,767 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=28230b65c0b592a41b759dc13b865232, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,37917,1731708686516; forceNewPlan=false, retain=false 2024-11-15T22:13:58,918 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=28230b65c0b592a41b759dc13b865232, regionState=OPENING, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:58,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=28230b65c0b592a41b759dc13b865232, ASSIGN because future has completed 2024-11-15T22:13:58,920 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516}] 2024-11-15T22:13:59,075 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:13:59,075 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7752): Opening region: {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:13:59,075 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 28230b65c0b592a41b759dc13b865232 2024-11-15T22:13:59,075 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:13:59,075 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7794): checking encryption for 28230b65c0b592a41b759dc13b865232 2024-11-15T22:13:59,075 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7797): checking classloading for 28230b65c0b592a41b759dc13b865232 2024-11-15T22:13:59,077 INFO [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 28230b65c0b592a41b759dc13b865232 2024-11-15T22:13:59,080 INFO [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28230b65c0b592a41b759dc13b865232 columnFamilyName A 2024-11-15T22:13:59,080 DEBUG [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:59,081 INFO [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] regionserver.HStore(327): Store=28230b65c0b592a41b759dc13b865232/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:13:59,081 INFO [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 28230b65c0b592a41b759dc13b865232 2024-11-15T22:13:59,083 INFO [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28230b65c0b592a41b759dc13b865232 columnFamilyName B 2024-11-15T22:13:59,083 DEBUG [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:59,084 INFO [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] regionserver.HStore(327): Store=28230b65c0b592a41b759dc13b865232/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:13:59,084 INFO [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 28230b65c0b592a41b759dc13b865232 2024-11-15T22:13:59,085 INFO [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28230b65c0b592a41b759dc13b865232 columnFamilyName C 2024-11-15T22:13:59,086 DEBUG [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:13:59,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T22:13:59,087 INFO [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] regionserver.HStore(327): Store=28230b65c0b592a41b759dc13b865232/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:13:59,087 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1038): replaying wal for 28230b65c0b592a41b759dc13b865232 2024-11-15T22:13:59,087 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232 2024-11-15T22:13:59,088 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232 2024-11-15T22:13:59,090 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1048): stopping wal replay for 28230b65c0b592a41b759dc13b865232 2024-11-15T22:13:59,090 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1060): Cleaning up temporary data for 28230b65c0b592a41b759dc13b865232 2024-11-15T22:13:59,091 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:13:59,092 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1093): writing seq id for 28230b65c0b592a41b759dc13b865232 2024-11-15T22:13:59,095 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:13:59,095 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1114): Opened 28230b65c0b592a41b759dc13b865232; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73361585, jitterRate=0.09317280352115631}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:13:59,095 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 28230b65c0b592a41b759dc13b865232 2024-11-15T22:13:59,096 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1006): Region open journal for 28230b65c0b592a41b759dc13b865232: Running coprocessor pre-open hook at 1731708839075Writing region info on filesystem at 1731708839075Initializing all the Stores at 1731708839076 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708839076Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708839077 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708839077Cleaning up temporary data from old regions at 1731708839090 (+13 ms)Running coprocessor post-open hooks at 1731708839095 (+5 ms)Region opened successfully at 1731708839096 (+1 ms) 2024-11-15T22:13:59,096 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., pid=150, masterSystemTime=1731708839071 2024-11-15T22:13:59,098 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:13:59,098 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:13:59,099 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=28230b65c0b592a41b759dc13b865232, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:13:59,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 because future has completed 2024-11-15T22:13:59,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-15T22:13:59,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 in 181 msec 2024-11-15T22:13:59,106 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-15T22:13:59,106 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=28230b65c0b592a41b759dc13b865232, ASSIGN in 337 msec 2024-11-15T22:13:59,106 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T22:13:59,107 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708839106"}]},"ts":"1731708839106"} 2024-11-15T22:13:59,108 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T22:13:59,109 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T22:13:59,110 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1640 sec 2024-11-15T22:14:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T22:14:00,097 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:14:00,098 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b9d0221, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:00,098 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:14:00,098 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:14:00,099 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:14:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:14:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:14:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59bb0278, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:14:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:14:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:00,100 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:14:00,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34ff0afa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:00,101 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:14:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:14:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:14:00,103 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:14:00,103 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T22:14:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is ae3311ea5ccb,37303,1731708685620 2024-11-15T22:14:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@40cc7dee 2024-11-15T22:14:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T22:14:00,104 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T22:14:00,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T22:14:00,106 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:14:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T22:14:00,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742381_1557 (size=995) 2024-11-15T22:14:00,516 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-15T22:14:00,516 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-15T22:14:00,518 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T22:14:00,520 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=28230b65c0b592a41b759dc13b865232, REOPEN/MOVE}] 2024-11-15T22:14:00,520 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=28230b65c0b592a41b759dc13b865232, REOPEN/MOVE 2024-11-15T22:14:00,521 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=28230b65c0b592a41b759dc13b865232, regionState=CLOSING, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:00,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=28230b65c0b592a41b759dc13b865232, REOPEN/MOVE because future has completed 2024-11-15T22:14:00,522 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T22:14:00,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure 28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516}] 2024-11-15T22:14:00,674 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(122): Close 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:00,674 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T22:14:00,675 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1722): Closing 28230b65c0b592a41b759dc13b865232, disabling compactions & flushes 2024-11-15T22:14:00,675 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:00,675 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:00,675 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. after waiting 0 ms 2024-11-15T22:14:00,675 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:00,678 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:00,678 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:00,678 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1676): Region close journal for 28230b65c0b592a41b759dc13b865232: Waiting for close lock at 1731708840674Running coprocessor pre-close hooks at 1731708840675 (+1 ms)Disabling compacts and flushes for region at 1731708840675Disabling writes for close at 1731708840675Writing region close event to WAL at 1731708840675Running coprocessor post-close hooks at 1731708840678 (+3 ms)Closed at 1731708840678 2024-11-15T22:14:00,678 WARN [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegionServer(3299): Not adding moved region record: 28230b65c0b592a41b759dc13b865232 to self. 2024-11-15T22:14:00,680 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(157): Closed 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:00,680 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=28230b65c0b592a41b759dc13b865232, regionState=CLOSED 2024-11-15T22:14:00,681 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure 28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 because future has completed 2024-11-15T22:14:00,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-15T22:14:00,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; CloseRegionProcedure 28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 in 160 msec 2024-11-15T22:14:00,684 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=28230b65c0b592a41b759dc13b865232, REOPEN/MOVE; state=CLOSED, location=ae3311ea5ccb,37917,1731708686516; forceNewPlan=false, retain=true 2024-11-15T22:14:00,834 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=28230b65c0b592a41b759dc13b865232, regionState=OPENING, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:00,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=28230b65c0b592a41b759dc13b865232, REOPEN/MOVE because future has completed 2024-11-15T22:14:00,836 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516}] 2024-11-15T22:14:00,991 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:00,991 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7752): Opening region: {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:14:00,991 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:00,991 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:00,991 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7794): checking encryption for 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:00,991 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7797): checking classloading for 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:00,992 INFO [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:00,993 INFO [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28230b65c0b592a41b759dc13b865232 columnFamilyName A 2024-11-15T22:14:00,994 DEBUG [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:00,994 INFO [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] regionserver.HStore(327): Store=28230b65c0b592a41b759dc13b865232/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:00,995 INFO [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:00,995 INFO [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28230b65c0b592a41b759dc13b865232 columnFamilyName B 2024-11-15T22:14:00,995 DEBUG [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:00,995 INFO [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] regionserver.HStore(327): Store=28230b65c0b592a41b759dc13b865232/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:00,996 INFO [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:00,996 INFO [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28230b65c0b592a41b759dc13b865232 columnFamilyName C 2024-11-15T22:14:00,996 DEBUG [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:00,996 INFO [StoreOpener-28230b65c0b592a41b759dc13b865232-1 {}] regionserver.HStore(327): Store=28230b65c0b592a41b759dc13b865232/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:00,996 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1038): replaying wal for 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:00,997 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:00,998 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:00,998 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1048): stopping wal replay for 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:00,998 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1060): Cleaning up temporary data for 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:00,998 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T22:14:00,999 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1093): writing seq id for 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:01,000 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1114): Opened 28230b65c0b592a41b759dc13b865232; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65050764, jitterRate=-0.03066807985305786}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T22:14:01,000 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:01,000 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1006): Region open journal for 28230b65c0b592a41b759dc13b865232: Running coprocessor pre-open hook at 1731708840991Writing region info on filesystem at 1731708840991Initializing all the Stores at 1731708840992 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708840992Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708840992Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708840992Cleaning up temporary data from old regions at 1731708840998 (+6 ms)Running coprocessor post-open hooks at 1731708841000 (+2 ms)Region opened successfully at 1731708841000 2024-11-15T22:14:01,001 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., pid=155, masterSystemTime=1731708840988 2024-11-15T22:14:01,003 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:01,003 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:01,003 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=28230b65c0b592a41b759dc13b865232, regionState=OPEN, openSeqNum=5, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 because future has completed 2024-11-15T22:14:01,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=153 2024-11-15T22:14:01,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure 28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 in 169 msec 2024-11-15T22:14:01,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-15T22:14:01,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=28230b65c0b592a41b759dc13b865232, REOPEN/MOVE in 486 msec 2024-11-15T22:14:01,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-15T22:14:01,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 490 msec 2024-11-15T22:14:01,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 904 msec 2024-11-15T22:14:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-15T22:14:01,012 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:14:01,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a3fe0ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,012 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:14:01,012 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:14:01,013 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:14:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:14:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:14:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c344a03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:14:01,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:14:01,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:01,014 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:14:01,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aacaceb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9e20296, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,015 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:14:01,015 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:14:01,016 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:14:01,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:14:01,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:14:01,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b5a4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:14:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:14:01,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:01,017 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:14:01,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d2fa3bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c56a33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:14:01,018 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:14:01,019 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:14:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:14:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:14:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34fa25e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:14:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:14:01,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:01,020 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:14:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1460749b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,021 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@362a517d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,021 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:14:01,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:14:01,021 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:14:01,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:14:01,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:14:01,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5404d503, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:14:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:14:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:01,022 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:14:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f4f40f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1001aa8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,023 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:14:01,023 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:14:01,024 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:14:01,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:14:01,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:14:01,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b1743cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:14:01,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:14:01,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:01,025 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:14:01,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c73454f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,026 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76841744, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,026 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:14:01,026 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:14:01,026 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:14:01,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:14:01,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:14:01,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bfea5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:14:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:14:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:01,027 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:14:01,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34f1de52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78ab0eab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:14:01,028 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:14:01,029 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:14:01,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:14:01,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:14:01,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6335e366, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:14:01,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:14:01,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:01,030 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:14:01,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ad4a62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b86f115, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,031 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:14:01,031 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:14:01,031 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:14:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:14:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:14:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57ebc6be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:14:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:14:01,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:01,033 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:14:01,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@400fec1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fe4550b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,034 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:14:01,034 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:14:01,034 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:14:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:14:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:14:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eff6d8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:14:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:14:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:01,035 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:14:01,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f8de63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@798c059e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,37303,-1 for getting cluster id 2024-11-15T22:14:01,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:14:01,037 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f' 2024-11-15T22:14:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:14:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cd1d2642-6ff0-4b7b-bd4d-f57a96c1d10f" 2024-11-15T22:14:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bdc5ca3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,37303,-1] 2024-11-15T22:14:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:14:01,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:01,038 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:14:01,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55bafe15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:01,039 DEBUG [Thread-2493 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:14:01,039 DEBUG [Thread-2486 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:14:01,039 DEBUG [Thread-2497 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:14:01,039 DEBUG [Thread-2478 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:14:01,039 DEBUG [Thread-2480 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:14:01,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:14:01,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:14:01,039 DEBUG [Thread-2495 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:14:01,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:14:01,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:14:01,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:14:01,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:14:01,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:14:01,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:14:01,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:14:01,040 DEBUG [Thread-2482 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:14:01,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:14:01,041 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:14:01,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:14:01,041 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:14:01,041 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:14:01,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:14:01,041 DEBUG [Thread-2491 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:14:01,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:14:01,041 DEBUG [Thread-2484 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:14:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:14:01,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:14:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:14:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:14:01,042 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:14:01,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:14:01,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:14:01,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:14:01,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:14:01,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:14:01,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:14:01,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:14:01,043 DEBUG [Thread-2489 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:14:01,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:14:01,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,37917,1731708686516, seqNum=-1] 2024-11-15T22:14:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-15T22:14:01,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:14:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T22:14:01,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:14:01,045 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:14:01,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:14:01,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:14:01,046 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:14:01,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:14:01,047 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:14:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:14:01,047 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:14:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:14:01,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:14:01,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:14:01,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5] 2024-11-15T22:14:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:01,052 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:14:01,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T22:14:01,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708901075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708901076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f8258c94c8f242bca099827e34ba6f54_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708841049/Put/seqid=0 2024-11-15T22:14:01,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708901079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708901080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708901080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742382_1558 (size=14594) 2024-11-15T22:14:01,103 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:01,108 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f8258c94c8f242bca099827e34ba6f54_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f8258c94c8f242bca099827e34ba6f54_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:01,111 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/f75b6aa148df4df58ad0eb64608a1cf8, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:01,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/f75b6aa148df4df58ad0eb64608a1cf8 is 175, key is test_row_0/A:col10/1731708841049/Put/seqid=0 2024-11-15T22:14:01,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742383_1559 (size=39549) 2024-11-15T22:14:01,116 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/f75b6aa148df4df58ad0eb64608a1cf8 2024-11-15T22:14:01,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/9c6967e198a14eb4abec8d77f84d17f8 is 50, key is test_row_0/B:col10/1731708841049/Put/seqid=0 2024-11-15T22:14:01,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742384_1560 (size=12001) 2024-11-15T22:14:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T22:14:01,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/9c6967e198a14eb4abec8d77f84d17f8 2024-11-15T22:14:01,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/f8c125714cae43f3bbe31111d5193647 is 50, key is test_row_0/C:col10/1731708841049/Put/seqid=0 2024-11-15T22:14:01,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708901187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708901187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708901187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742385_1561 (size=12001) 2024-11-15T22:14:01,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/f8c125714cae43f3bbe31111d5193647 2024-11-15T22:14:01,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708901196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708901196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T22:14:01,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:01,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:01,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:01,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:01,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:01,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/f75b6aa148df4df58ad0eb64608a1cf8 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/f75b6aa148df4df58ad0eb64608a1cf8 2024-11-15T22:14:01,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/f75b6aa148df4df58ad0eb64608a1cf8, entries=200, sequenceid=15, filesize=38.6 K 2024-11-15T22:14:01,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/9c6967e198a14eb4abec8d77f84d17f8 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9c6967e198a14eb4abec8d77f84d17f8 2024-11-15T22:14:01,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9c6967e198a14eb4abec8d77f84d17f8, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T22:14:01,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/f8c125714cae43f3bbe31111d5193647 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/f8c125714cae43f3bbe31111d5193647 2024-11-15T22:14:01,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/f8c125714cae43f3bbe31111d5193647, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T22:14:01,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 28230b65c0b592a41b759dc13b865232 in 176ms, sequenceid=15, compaction requested=false 2024-11-15T22:14:01,229 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T22:14:01,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:01,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T22:14:01,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:01,355 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T22:14:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T22:14:01,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d8de4441d2b64af285b5ca980914901d_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708841073/Put/seqid=0 2024-11-15T22:14:01,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742386_1562 (size=12154) 2024-11-15T22:14:01,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:01,378 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d8de4441d2b64af285b5ca980914901d_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d8de4441d2b64af285b5ca980914901d_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:01,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/828313d5abbe4977bc77112d2ba8f2e2, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:01,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/828313d5abbe4977bc77112d2ba8f2e2 is 175, key is test_row_0/A:col10/1731708841073/Put/seqid=0 2024-11-15T22:14:01,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742387_1563 (size=30955) 2024-11-15T22:14:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:01,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:01,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708901405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708901405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708901406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708901407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708901409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708901516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708901516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708901516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T22:14:01,692 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T22:14:01,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708901717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708901717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708901726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708901726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:01,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708901726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:01,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:01,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:01,786 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/828313d5abbe4977bc77112d2ba8f2e2 2024-11-15T22:14:01,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/9afc4bbc92b94c8282ff6282ff6a6481 is 50, key is test_row_0/B:col10/1731708841073/Put/seqid=0 2024-11-15T22:14:01,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742388_1564 (size=12001) 2024-11-15T22:14:01,797 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/9afc4bbc92b94c8282ff6282ff6a6481 2024-11-15T22:14:01,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/8b8986a799834c7a9c977ff0814999b9 is 50, key is test_row_0/C:col10/1731708841073/Put/seqid=0 2024-11-15T22:14:01,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742389_1565 (size=12001) 2024-11-15T22:14:01,809 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/8b8986a799834c7a9c977ff0814999b9 2024-11-15T22:14:01,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/828313d5abbe4977bc77112d2ba8f2e2 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/828313d5abbe4977bc77112d2ba8f2e2 2024-11-15T22:14:01,817 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/828313d5abbe4977bc77112d2ba8f2e2, entries=150, sequenceid=41, filesize=30.2 K 2024-11-15T22:14:01,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/9afc4bbc92b94c8282ff6282ff6a6481 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9afc4bbc92b94c8282ff6282ff6a6481 2024-11-15T22:14:01,821 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9afc4bbc92b94c8282ff6282ff6a6481, entries=150, sequenceid=41, filesize=11.7 K 2024-11-15T22:14:01,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/8b8986a799834c7a9c977ff0814999b9 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/8b8986a799834c7a9c977ff0814999b9 2024-11-15T22:14:01,826 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/8b8986a799834c7a9c977ff0814999b9, entries=150, sequenceid=41, filesize=11.7 K 2024-11-15T22:14:01,827 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 28230b65c0b592a41b759dc13b865232 in 472ms, sequenceid=41, compaction requested=false 2024-11-15T22:14:01,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:01,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:01,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-15T22:14:01,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-15T22:14:01,831 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-15T22:14:01,831 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 782 msec 2024-11-15T22:14:01,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 789 msec 2024-11-15T22:14:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:02,039 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:14:02,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T22:14:02,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111539efa8ad7801483eb02e24d422449178_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708841406/Put/seqid=0 2024-11-15T22:14:02,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742390_1566 (size=14594) 2024-11-15T22:14:02,055 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:02,058 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111539efa8ad7801483eb02e24d422449178_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111539efa8ad7801483eb02e24d422449178_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:02,059 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/573426b6b06140f4b87680440e5f3c96, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:02,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/573426b6b06140f4b87680440e5f3c96 is 175, key is test_row_0/A:col10/1731708841406/Put/seqid=0 2024-11-15T22:14:02,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742391_1567 (size=39549) 2024-11-15T22:14:02,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708902079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708902079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:02,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708902080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:02,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:02,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T22:14:02,177 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:14:02,177 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:14:02,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-15T22:14:02,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T22:14:02,185 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:14:02,186 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:14:02,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:14:02,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708902197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:02,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:02,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708902197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:02,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:02,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708902197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:02,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:02,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708902226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:02,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:02,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708902226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:02,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T22:14:02,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T22:14:02,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:02,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:02,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:02,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:02,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:02,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:02,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708902406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708902406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:02,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708902407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:02,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:02,474 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/573426b6b06140f4b87680440e5f3c96 2024-11-15T22:14:02,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/027f192a544d4dbf965082e14f020256 is 50, key is test_row_0/B:col10/1731708841406/Put/seqid=0 2024-11-15T22:14:02,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T22:14:02,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:02,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:02,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:02,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:02,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:02,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:02,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T22:14:02,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742392_1568 (size=12001) 2024-11-15T22:14:02,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T22:14:02,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:02,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:02,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:02,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:02,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:02,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:02,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708902716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:02,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:02,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708902716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:02,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:02,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:02,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708902726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:02,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:02,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:02,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T22:14:02,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:02,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:02,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:02,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:02,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:02,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:02,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T22:14:02,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/027f192a544d4dbf965082e14f020256 2024-11-15T22:14:02,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T22:14:02,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:02,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:02,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:02,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:02,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:02,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/7e03166b1cfd4df9994b7ce90a4b8dbb is 50, key is test_row_0/C:col10/1731708841406/Put/seqid=0 2024-11-15T22:14:02,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742393_1569 (size=12001) 2024-11-15T22:14:02,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/7e03166b1cfd4df9994b7ce90a4b8dbb 2024-11-15T22:14:03,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/573426b6b06140f4b87680440e5f3c96 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/573426b6b06140f4b87680440e5f3c96 2024-11-15T22:14:03,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/573426b6b06140f4b87680440e5f3c96, entries=200, sequenceid=53, filesize=38.6 K 2024-11-15T22:14:03,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/027f192a544d4dbf965082e14f020256 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/027f192a544d4dbf965082e14f020256 2024-11-15T22:14:03,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/027f192a544d4dbf965082e14f020256, entries=150, sequenceid=53, filesize=11.7 K 2024-11-15T22:14:03,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/7e03166b1cfd4df9994b7ce90a4b8dbb as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/7e03166b1cfd4df9994b7ce90a4b8dbb 2024-11-15T22:14:03,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/7e03166b1cfd4df9994b7ce90a4b8dbb, entries=150, sequenceid=53, filesize=11.7 K 2024-11-15T22:14:03,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 28230b65c0b592a41b759dc13b865232 in 989ms, sequenceid=53, compaction requested=true 2024-11-15T22:14:03,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:03,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:14:03,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:03,028 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:14:03,028 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:14:03,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:14:03,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:03,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:14:03,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:14:03,030 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:14:03,030 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/B is initiating minor compaction (all files) 2024-11-15T22:14:03,030 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:14:03,030 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/B in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:03,030 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/A is initiating minor compaction (all files) 2024-11-15T22:14:03,030 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9c6967e198a14eb4abec8d77f84d17f8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9afc4bbc92b94c8282ff6282ff6a6481, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/027f192a544d4dbf965082e14f020256] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=35.2 K 2024-11-15T22:14:03,030 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/A in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:03,030 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/f75b6aa148df4df58ad0eb64608a1cf8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/828313d5abbe4977bc77112d2ba8f2e2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/573426b6b06140f4b87680440e5f3c96] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=107.5 K 2024-11-15T22:14:03,030 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:03,030 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/f75b6aa148df4df58ad0eb64608a1cf8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/828313d5abbe4977bc77112d2ba8f2e2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/573426b6b06140f4b87680440e5f3c96] 2024-11-15T22:14:03,031 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting f75b6aa148df4df58ad0eb64608a1cf8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731708841046 2024-11-15T22:14:03,031 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c6967e198a14eb4abec8d77f84d17f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731708841046 2024-11-15T22:14:03,031 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 828313d5abbe4977bc77112d2ba8f2e2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731708841073 2024-11-15T22:14:03,031 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 9afc4bbc92b94c8282ff6282ff6a6481, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731708841073 2024-11-15T22:14:03,032 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 573426b6b06140f4b87680440e5f3c96, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731708841403 2024-11-15T22:14:03,032 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 027f192a544d4dbf965082e14f020256, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731708841403 2024-11-15T22:14:03,047 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:03,050 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#B#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:03,050 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/2db242e14d2444a5b05de7a099ca14b9 is 50, key is test_row_0/B:col10/1731708841406/Put/seqid=0 2024-11-15T22:14:03,051 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115cfdaac08c7ed4d5b99e2fb750ee175e5_28230b65c0b592a41b759dc13b865232 store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:03,053 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115cfdaac08c7ed4d5b99e2fb750ee175e5_28230b65c0b592a41b759dc13b865232, store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:03,054 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cfdaac08c7ed4d5b99e2fb750ee175e5_28230b65c0b592a41b759dc13b865232 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:03,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742394_1570 (size=12104) 2024-11-15T22:14:03,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742395_1571 (size=4469) 2024-11-15T22:14:03,073 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#A#compaction#473 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:03,073 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/709da9ef645b462cb52c9e9cb1a2abe9 is 175, key is test_row_0/A:col10/1731708841406/Put/seqid=0 2024-11-15T22:14:03,089 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/2db242e14d2444a5b05de7a099ca14b9 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/2db242e14d2444a5b05de7a099ca14b9 2024-11-15T22:14:03,100 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28230b65c0b592a41b759dc13b865232/B of 28230b65c0b592a41b759dc13b865232 into 2db242e14d2444a5b05de7a099ca14b9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:03,100 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:03,100 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/B, priority=13, startTime=1731708843028; duration=0sec 2024-11-15T22:14:03,100 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:14:03,100 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:B 2024-11-15T22:14:03,100 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:14:03,101 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:14:03,102 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/C is initiating minor compaction (all files) 2024-11-15T22:14:03,102 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/C in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:03,102 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/f8c125714cae43f3bbe31111d5193647, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/8b8986a799834c7a9c977ff0814999b9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/7e03166b1cfd4df9994b7ce90a4b8dbb] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=35.2 K 2024-11-15T22:14:03,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742396_1572 (size=31058) 2024-11-15T22:14:03,103 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting f8c125714cae43f3bbe31111d5193647, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731708841046 2024-11-15T22:14:03,104 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b8986a799834c7a9c977ff0814999b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731708841073 2024-11-15T22:14:03,104 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e03166b1cfd4df9994b7ce90a4b8dbb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731708841403 2024-11-15T22:14:03,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T22:14:03,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:03,109 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T22:14:03,112 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/709da9ef645b462cb52c9e9cb1a2abe9 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/709da9ef645b462cb52c9e9cb1a2abe9 2024-11-15T22:14:03,119 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28230b65c0b592a41b759dc13b865232/A of 28230b65c0b592a41b759dc13b865232 into 709da9ef645b462cb52c9e9cb1a2abe9(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:03,119 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:03,119 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/A, priority=13, startTime=1731708843028; duration=0sec 2024-11-15T22:14:03,119 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:03,119 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:A 2024-11-15T22:14:03,123 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#C#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:03,124 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/e8a10c72bd354098ac8afa58c2c62d60 is 50, key is test_row_0/C:col10/1731708841406/Put/seqid=0 2024-11-15T22:14:03,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115455cb90e008f462bb6be0568d15b8b88_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708842069/Put/seqid=0 2024-11-15T22:14:03,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742397_1573 (size=12104) 2024-11-15T22:14:03,145 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/e8a10c72bd354098ac8afa58c2c62d60 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/e8a10c72bd354098ac8afa58c2c62d60 2024-11-15T22:14:03,150 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28230b65c0b592a41b759dc13b865232/C of 28230b65c0b592a41b759dc13b865232 into e8a10c72bd354098ac8afa58c2c62d60(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:03,150 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:03,151 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/C, priority=13, startTime=1731708843028; duration=0sec 2024-11-15T22:14:03,151 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:03,151 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:C 2024-11-15T22:14:03,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742398_1574 (size=12154) 2024-11-15T22:14:03,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:03,159 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115455cb90e008f462bb6be0568d15b8b88_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115455cb90e008f462bb6be0568d15b8b88_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:03,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/d656b325692749dfa185c1a3dbaaa419, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:03,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/d656b325692749dfa185c1a3dbaaa419 is 175, key is test_row_0/A:col10/1731708842069/Put/seqid=0 2024-11-15T22:14:03,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742399_1575 (size=30955) 2024-11-15T22:14:03,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:03,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708903242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:03,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708903246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:03,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708903248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:03,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708903250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708903250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T22:14:03,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708903356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:03,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708903357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:03,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708903357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:03,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708903357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:03,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:03,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708903366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:03,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:03,567 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/d656b325692749dfa185c1a3dbaaa419 2024-11-15T22:14:03,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/9db2e21db9e24d2bae78df2d2c7602d6 is 50, key is test_row_0/B:col10/1731708842069/Put/seqid=0 2024-11-15T22:14:03,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708903574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:03,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708903574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:03,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708903574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:03,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:03,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708903576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:03,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:03,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708903577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:03,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742400_1576 (size=12001) 2024-11-15T22:14:03,600 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/9db2e21db9e24d2bae78df2d2c7602d6 2024-11-15T22:14:03,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/4f54e8cfada947ab9f06714561aca4d9 is 50, key is test_row_0/C:col10/1731708842069/Put/seqid=0 2024-11-15T22:14:03,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742401_1577 (size=12001) 2024-11-15T22:14:03,641 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/4f54e8cfada947ab9f06714561aca4d9 2024-11-15T22:14:03,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/d656b325692749dfa185c1a3dbaaa419 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/d656b325692749dfa185c1a3dbaaa419 2024-11-15T22:14:03,651 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/d656b325692749dfa185c1a3dbaaa419, entries=150, sequenceid=78, filesize=30.2 K 2024-11-15T22:14:03,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/9db2e21db9e24d2bae78df2d2c7602d6 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9db2e21db9e24d2bae78df2d2c7602d6 2024-11-15T22:14:03,659 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9db2e21db9e24d2bae78df2d2c7602d6, entries=150, sequenceid=78, filesize=11.7 K 2024-11-15T22:14:03,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/4f54e8cfada947ab9f06714561aca4d9 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/4f54e8cfada947ab9f06714561aca4d9 2024-11-15T22:14:03,674 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/4f54e8cfada947ab9f06714561aca4d9, entries=150, sequenceid=78, filesize=11.7 K 2024-11-15T22:14:03,677 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 28230b65c0b592a41b759dc13b865232 in 568ms, sequenceid=78, compaction requested=false 2024-11-15T22:14:03,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:03,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:03,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-15T22:14:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-15T22:14:03,691 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-15T22:14:03,691 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4970 sec 2024-11-15T22:14:03,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.5150 sec 2024-11-15T22:14:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:03,892 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:14:03,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T22:14:03,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159af0693a7d704cc4b3a122b72137b039_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708843892/Put/seqid=0 2024-11-15T22:14:03,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742402_1578 (size=12154) 2024-11-15T22:14:03,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:03,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708903966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:03,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:03,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708903971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:03,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:03,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708903972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:03,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:03,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708903972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:03,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:03,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:03,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:03,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708903973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:04,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708904076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:04,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708904086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:04,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708904087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:04,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708904087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:04,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708904087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:04,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708904286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:04,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708904296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708904296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:04,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708904297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:04,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708904297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:04,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T22:14:04,326 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:14:04,327 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:14:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-15T22:14:04,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T22:14:04,329 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:14:04,330 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:14:04,330 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:14:04,330 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:04,334 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159af0693a7d704cc4b3a122b72137b039_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159af0693a7d704cc4b3a122b72137b039_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:04,335 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/3ca75e2e5d25469cadf28de960f34021, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:04,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/3ca75e2e5d25469cadf28de960f34021 is 175, key is test_row_0/A:col10/1731708843892/Put/seqid=0 2024-11-15T22:14:04,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742403_1579 (size=30955) 2024-11-15T22:14:04,341 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/3ca75e2e5d25469cadf28de960f34021 2024-11-15T22:14:04,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/9e72daf5f75a42a9b5deac0b3946a8d1 is 50, key is test_row_0/B:col10/1731708843892/Put/seqid=0 2024-11-15T22:14:04,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742404_1580 (size=12001) 2024-11-15T22:14:04,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T22:14:04,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T22:14:04,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:04,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:04,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:04,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:04,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:04,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:04,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:04,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708904597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:04,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708904606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708904606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708904606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708904606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:04,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T22:14:04,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:04,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:04,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:04,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:04,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:04,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T22:14:04,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/9e72daf5f75a42a9b5deac0b3946a8d1 2024-11-15T22:14:04,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/24c36fe63b2c44d8b0cc067d8bcde189 is 50, key is test_row_0/C:col10/1731708843892/Put/seqid=0 2024-11-15T22:14:04,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742405_1581 (size=12001) 2024-11-15T22:14:04,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T22:14:04,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:04,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:04,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:04,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:04,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:04,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:04,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T22:14:04,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:04,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:04,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:04,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:04,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:04,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:04,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T22:14:05,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T22:14:05,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:05,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:05,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:05,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:05,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:05,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708905107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:05,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:05,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:05,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708905116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:05,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:05,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:05,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:05,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708905116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:05,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708905116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:05,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708905126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:05,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:05,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:05,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:05,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/24c36fe63b2c44d8b0cc067d8bcde189 2024-11-15T22:14:05,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/3ca75e2e5d25469cadf28de960f34021 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/3ca75e2e5d25469cadf28de960f34021 2024-11-15T22:14:05,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/3ca75e2e5d25469cadf28de960f34021, entries=150, sequenceid=97, filesize=30.2 K 2024-11-15T22:14:05,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/9e72daf5f75a42a9b5deac0b3946a8d1 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9e72daf5f75a42a9b5deac0b3946a8d1 2024-11-15T22:14:05,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9e72daf5f75a42a9b5deac0b3946a8d1, entries=150, sequenceid=97, filesize=11.7 K 2024-11-15T22:14:05,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/24c36fe63b2c44d8b0cc067d8bcde189 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/24c36fe63b2c44d8b0cc067d8bcde189 2024-11-15T22:14:05,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/24c36fe63b2c44d8b0cc067d8bcde189, entries=150, sequenceid=97, filesize=11.7 K 2024-11-15T22:14:05,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 28230b65c0b592a41b759dc13b865232 in 1294ms, sequenceid=97, compaction requested=true 2024-11-15T22:14:05,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:05,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:14:05,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:05,186 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:14:05,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:14:05,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:05,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:14:05,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:14:05,186 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:14:05,187 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:14:05,187 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/A is initiating minor compaction (all files) 2024-11-15T22:14:05,187 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/A in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:05,188 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/709da9ef645b462cb52c9e9cb1a2abe9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/d656b325692749dfa185c1a3dbaaa419, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/3ca75e2e5d25469cadf28de960f34021] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=90.8 K 2024-11-15T22:14:05,188 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:05,188 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/709da9ef645b462cb52c9e9cb1a2abe9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/d656b325692749dfa185c1a3dbaaa419, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/3ca75e2e5d25469cadf28de960f34021] 2024-11-15T22:14:05,188 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:14:05,188 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/B is initiating minor compaction (all files) 2024-11-15T22:14:05,188 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/B in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:05,188 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/2db242e14d2444a5b05de7a099ca14b9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9db2e21db9e24d2bae78df2d2c7602d6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9e72daf5f75a42a9b5deac0b3946a8d1] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=35.3 K 2024-11-15T22:14:05,189 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 709da9ef645b462cb52c9e9cb1a2abe9, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731708841403 2024-11-15T22:14:05,189 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 2db242e14d2444a5b05de7a099ca14b9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731708841403 2024-11-15T22:14:05,189 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 9db2e21db9e24d2bae78df2d2c7602d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731708842069 2024-11-15T22:14:05,189 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting d656b325692749dfa185c1a3dbaaa419, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731708842069 2024-11-15T22:14:05,189 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e72daf5f75a42a9b5deac0b3946a8d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731708843241 2024-11-15T22:14:05,189 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ca75e2e5d25469cadf28de960f34021, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731708843241 2024-11-15T22:14:05,198 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#B#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:05,198 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:05,198 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/ea1105a92fb64681a9db70c1cd47b537 is 50, key is test_row_0/B:col10/1731708843892/Put/seqid=0 2024-11-15T22:14:05,202 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115d0c74d49d2cb44bea253c411ab73965b_28230b65c0b592a41b759dc13b865232 store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:05,204 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115d0c74d49d2cb44bea253c411ab73965b_28230b65c0b592a41b759dc13b865232, store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:05,204 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d0c74d49d2cb44bea253c411ab73965b_28230b65c0b592a41b759dc13b865232 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:05,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742406_1582 (size=12207) 2024-11-15T22:14:05,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742407_1583 (size=4469) 2024-11-15T22:14:05,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T22:14:05,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:05,255 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T22:14:05,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111590a91d28f4d246639804e7c65ad6ad53_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708843963/Put/seqid=0 2024-11-15T22:14:05,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742408_1584 (size=12154) 2024-11-15T22:14:05,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:05,277 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111590a91d28f4d246639804e7c65ad6ad53_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111590a91d28f4d246639804e7c65ad6ad53_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:05,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/35a5ca162c1e422ab332cfc12706b4a6, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:05,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/35a5ca162c1e422ab332cfc12706b4a6 is 175, key is test_row_0/A:col10/1731708843963/Put/seqid=0 2024-11-15T22:14:05,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742409_1585 (size=30955) 2024-11-15T22:14:05,286 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/35a5ca162c1e422ab332cfc12706b4a6 2024-11-15T22:14:05,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/34d01dbfce924f23bae1f779917730f7 is 50, key is test_row_0/B:col10/1731708843963/Put/seqid=0 2024-11-15T22:14:05,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742410_1586 (size=12001) 2024-11-15T22:14:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T22:14:05,622 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#A#compaction#483 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:05,623 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/64dfa8c256d843d79ad658a3af55aa9d is 175, key is test_row_0/A:col10/1731708843892/Put/seqid=0 2024-11-15T22:14:05,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742411_1587 (size=31161) 2024-11-15T22:14:05,634 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/ea1105a92fb64681a9db70c1cd47b537 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/ea1105a92fb64681a9db70c1cd47b537 2024-11-15T22:14:05,644 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28230b65c0b592a41b759dc13b865232/B of 28230b65c0b592a41b759dc13b865232 into ea1105a92fb64681a9db70c1cd47b537(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:05,644 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:05,644 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/B, priority=13, startTime=1731708845186; duration=0sec 2024-11-15T22:14:05,644 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:14:05,644 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:B 2024-11-15T22:14:05,644 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:14:05,645 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:14:05,645 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/C is initiating minor compaction (all files) 2024-11-15T22:14:05,645 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/C in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:05,646 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/e8a10c72bd354098ac8afa58c2c62d60, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/4f54e8cfada947ab9f06714561aca4d9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/24c36fe63b2c44d8b0cc067d8bcde189] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=35.3 K 2024-11-15T22:14:05,647 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting e8a10c72bd354098ac8afa58c2c62d60, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731708841403 2024-11-15T22:14:05,647 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f54e8cfada947ab9f06714561aca4d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731708842069 2024-11-15T22:14:05,648 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 24c36fe63b2c44d8b0cc067d8bcde189, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731708843241 2024-11-15T22:14:05,660 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#C#compaction#486 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:05,661 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/cd2e56bcd9364bf0a5cefec8d317ed31 is 50, key is test_row_0/C:col10/1731708843892/Put/seqid=0 2024-11-15T22:14:05,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742412_1588 (size=12207) 2024-11-15T22:14:05,716 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/34d01dbfce924f23bae1f779917730f7 2024-11-15T22:14:05,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/00b0cc8f106d4ffa9dbef376e4ab7869 is 50, key is test_row_0/C:col10/1731708843963/Put/seqid=0 2024-11-15T22:14:05,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742413_1589 (size=12001) 2024-11-15T22:14:06,046 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/64dfa8c256d843d79ad658a3af55aa9d as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/64dfa8c256d843d79ad658a3af55aa9d 2024-11-15T22:14:06,051 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28230b65c0b592a41b759dc13b865232/A of 28230b65c0b592a41b759dc13b865232 into 64dfa8c256d843d79ad658a3af55aa9d(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:06,051 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:06,051 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/A, priority=13, startTime=1731708845186; duration=0sec 2024-11-15T22:14:06,051 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:06,051 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:A 2024-11-15T22:14:06,111 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/cd2e56bcd9364bf0a5cefec8d317ed31 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/cd2e56bcd9364bf0a5cefec8d317ed31 2024-11-15T22:14:06,128 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28230b65c0b592a41b759dc13b865232/C of 28230b65c0b592a41b759dc13b865232 into cd2e56bcd9364bf0a5cefec8d317ed31(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:06,129 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:06,129 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/C, priority=13, startTime=1731708845186; duration=0sec 2024-11-15T22:14:06,129 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:06,129 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:C 2024-11-15T22:14:06,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:06,135 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/00b0cc8f106d4ffa9dbef376e4ab7869 2024-11-15T22:14:06,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/35a5ca162c1e422ab332cfc12706b4a6 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/35a5ca162c1e422ab332cfc12706b4a6 2024-11-15T22:14:06,144 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/35a5ca162c1e422ab332cfc12706b4a6, entries=150, sequenceid=116, filesize=30.2 K 2024-11-15T22:14:06,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/34d01dbfce924f23bae1f779917730f7 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/34d01dbfce924f23bae1f779917730f7 2024-11-15T22:14:06,157 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/34d01dbfce924f23bae1f779917730f7, entries=150, sequenceid=116, filesize=11.7 K 2024-11-15T22:14:06,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/00b0cc8f106d4ffa9dbef376e4ab7869 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/00b0cc8f106d4ffa9dbef376e4ab7869 2024-11-15T22:14:06,160 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/00b0cc8f106d4ffa9dbef376e4ab7869, entries=150, sequenceid=116, filesize=11.7 K 2024-11-15T22:14:06,185 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=87.22 KB/89310 for 28230b65c0b592a41b759dc13b865232 in 930ms, sequenceid=116, compaction requested=false 2024-11-15T22:14:06,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:06,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:06,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-15T22:14:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:06,186 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:14:06,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T22:14:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-15T22:14:06,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-15T22:14:06,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8570 sec 2024-11-15T22:14:06,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.8630 sec 2024-11-15T22:14:06,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155380ec7fcc414eb6a11c33e65ea8fc15_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708846140/Put/seqid=0 2024-11-15T22:14:06,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742414_1590 (size=12304) 2024-11-15T22:14:06,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708906211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:06,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708906212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:06,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:06,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708906228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:06,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708906229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:06,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708906233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:06,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708906337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:06,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708906337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:06,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708906347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:06,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:06,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708906347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:06,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708906349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T22:14:06,478 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:14:06,478 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:14:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-15T22:14:06,480 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:14:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T22:14:06,481 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:14:06,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:14:06,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708906547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:06,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:06,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708906547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:06,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:06,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:06,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708906557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:06,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:06,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708906557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:06,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:06,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708906557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:06,558 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T22:14:06,599 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:06,613 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155380ec7fcc414eb6a11c33e65ea8fc15_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155380ec7fcc414eb6a11c33e65ea8fc15_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:06,621 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/b4e47fb9bc5841e99b7ec55d370f2c09, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:06,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/b4e47fb9bc5841e99b7ec55d370f2c09 is 175, key is test_row_0/A:col10/1731708846140/Put/seqid=0 2024-11-15T22:14:06,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T22:14:06,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:06,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:06,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:06,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:06,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:06,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742415_1591 (size=31105) 2024-11-15T22:14:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T22:14:06,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T22:14:06,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:06,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:06,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:06,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:06,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:06,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708906856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:06,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708906856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:06,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:06,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708906867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:06,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708906867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:06,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:06,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708906867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:06,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:06,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:06,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T22:14:06,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:06,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:06,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:06,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:06,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:07,050 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=138, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/b4e47fb9bc5841e99b7ec55d370f2c09 2024-11-15T22:14:07,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/23e9a5326530441e87acdffea310f3d4 is 50, key is test_row_0/B:col10/1731708846140/Put/seqid=0 2024-11-15T22:14:07,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742416_1592 (size=12151) 2024-11-15T22:14:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T22:14:07,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T22:14:07,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:07,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:07,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:07,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:07,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:07,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T22:14:07,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:07,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:07,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:07,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:07,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:07,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708907366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:07,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708907366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:07,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:07,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:07,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:07,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708907377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:07,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:07,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:07,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:07,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708907377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:07,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708907386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:07,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:07,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:07,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:07,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T22:14:07,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:07,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:07,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:07,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:07,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:07,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/23e9a5326530441e87acdffea310f3d4 2024-11-15T22:14:07,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/72c83be6ba45491e93e90cc104934898 is 50, key is test_row_0/C:col10/1731708846140/Put/seqid=0 2024-11-15T22:14:07,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742417_1593 (size=12151) 2024-11-15T22:14:07,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T22:14:07,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:07,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:07,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:07,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:07,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:07,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T22:14:07,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T22:14:07,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:07,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:07,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:07,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:07,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:07,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T22:14:07,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:07,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:07,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:07,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:07,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:07,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:07,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/72c83be6ba45491e93e90cc104934898 2024-11-15T22:14:07,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/b4e47fb9bc5841e99b7ec55d370f2c09 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/b4e47fb9bc5841e99b7ec55d370f2c09 2024-11-15T22:14:07,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/b4e47fb9bc5841e99b7ec55d370f2c09, entries=150, sequenceid=138, filesize=30.4 K 2024-11-15T22:14:07,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/23e9a5326530441e87acdffea310f3d4 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/23e9a5326530441e87acdffea310f3d4 2024-11-15T22:14:07,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/23e9a5326530441e87acdffea310f3d4, entries=150, sequenceid=138, filesize=11.9 K 2024-11-15T22:14:07,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/72c83be6ba45491e93e90cc104934898 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/72c83be6ba45491e93e90cc104934898 2024-11-15T22:14:07,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/72c83be6ba45491e93e90cc104934898, entries=150, sequenceid=138, filesize=11.9 K 2024-11-15T22:14:07,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 28230b65c0b592a41b759dc13b865232 in 1737ms, sequenceid=138, compaction requested=true 2024-11-15T22:14:07,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:07,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:14:07,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:07,923 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:14:07,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:14:07,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:07,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:14:07,923 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:14:07,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:14:07,924 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:14:07,924 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/B is initiating minor compaction (all files) 2024-11-15T22:14:07,924 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/B in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:07,925 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/ea1105a92fb64681a9db70c1cd47b537, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/34d01dbfce924f23bae1f779917730f7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/23e9a5326530441e87acdffea310f3d4] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=35.5 K 2024-11-15T22:14:07,925 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:14:07,925 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/A is initiating minor compaction (all files) 2024-11-15T22:14:07,925 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/A in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:07,925 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/64dfa8c256d843d79ad658a3af55aa9d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/35a5ca162c1e422ab332cfc12706b4a6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/b4e47fb9bc5841e99b7ec55d370f2c09] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=91.0 K 2024-11-15T22:14:07,925 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:07,925 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/64dfa8c256d843d79ad658a3af55aa9d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/35a5ca162c1e422ab332cfc12706b4a6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/b4e47fb9bc5841e99b7ec55d370f2c09] 2024-11-15T22:14:07,925 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting ea1105a92fb64681a9db70c1cd47b537, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731708843241 2024-11-15T22:14:07,926 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64dfa8c256d843d79ad658a3af55aa9d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731708843241 2024-11-15T22:14:07,926 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 34d01dbfce924f23bae1f779917730f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731708843963 2024-11-15T22:14:07,926 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35a5ca162c1e422ab332cfc12706b4a6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731708843963 2024-11-15T22:14:07,926 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 23e9a5326530441e87acdffea310f3d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731708846140 2024-11-15T22:14:07,926 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4e47fb9bc5841e99b7ec55d370f2c09, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731708846140 2024-11-15T22:14:07,940 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:07,941 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#B#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:07,941 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/74334bfa276843e884d74ee9109f629f is 50, key is test_row_0/B:col10/1731708846140/Put/seqid=0 2024-11-15T22:14:07,946 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115e9d587a1c09545dd8767eebbd9bfb85b_28230b65c0b592a41b759dc13b865232 store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:07,947 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115e9d587a1c09545dd8767eebbd9bfb85b_28230b65c0b592a41b759dc13b865232, store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:07,947 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e9d587a1c09545dd8767eebbd9bfb85b_28230b65c0b592a41b759dc13b865232 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:07,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742418_1594 (size=12459) 2024-11-15T22:14:07,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742419_1595 (size=4469) 2024-11-15T22:14:08,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T22:14:08,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:08,033 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-15T22:14:08,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155715f031355d4f309f840c3f0754fa29_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708846222/Put/seqid=0 2024-11-15T22:14:08,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742420_1596 (size=12304) 2024-11-15T22:14:08,375 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/74334bfa276843e884d74ee9109f629f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/74334bfa276843e884d74ee9109f629f 2024-11-15T22:14:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:08,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:08,382 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28230b65c0b592a41b759dc13b865232/B of 28230b65c0b592a41b759dc13b865232 into 74334bfa276843e884d74ee9109f629f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:08,382 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:08,382 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/B, priority=13, startTime=1731708847923; duration=0sec 2024-11-15T22:14:08,382 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:14:08,382 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:B 2024-11-15T22:14:08,382 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:14:08,384 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:14:08,384 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/C is initiating minor compaction (all files) 2024-11-15T22:14:08,384 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/C in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:08,384 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/cd2e56bcd9364bf0a5cefec8d317ed31, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/00b0cc8f106d4ffa9dbef376e4ab7869, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/72c83be6ba45491e93e90cc104934898] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=35.5 K 2024-11-15T22:14:08,385 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting cd2e56bcd9364bf0a5cefec8d317ed31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731708843241 2024-11-15T22:14:08,385 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 00b0cc8f106d4ffa9dbef376e4ab7869, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731708843963 2024-11-15T22:14:08,386 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 72c83be6ba45491e93e90cc104934898, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731708846140 2024-11-15T22:14:08,386 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#A#compaction#491 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:08,386 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/fd58df7155e84f2bb080addc2578867e is 175, key is test_row_0/A:col10/1731708846140/Put/seqid=0 2024-11-15T22:14:08,395 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#C#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:08,395 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/938fb2a9c49f4436a7ecdc25a55183c4 is 50, key is test_row_0/C:col10/1731708846140/Put/seqid=0 2024-11-15T22:14:08,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742421_1597 (size=31413) 2024-11-15T22:14:08,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742422_1598 (size=12459) 2024-11-15T22:14:08,407 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/938fb2a9c49f4436a7ecdc25a55183c4 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/938fb2a9c49f4436a7ecdc25a55183c4 2024-11-15T22:14:08,412 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28230b65c0b592a41b759dc13b865232/C of 28230b65c0b592a41b759dc13b865232 into 938fb2a9c49f4436a7ecdc25a55183c4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:08,412 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:08,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:08,412 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/C, priority=13, startTime=1731708847923; duration=0sec 2024-11-15T22:14:08,412 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708908404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:08,412 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:C 2024-11-15T22:14:08,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708908406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:08,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708908410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,417 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:08,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:08,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708908412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708908412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:08,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:08,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:08,493 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155715f031355d4f309f840c3f0754fa29_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155715f031355d4f309f840c3f0754fa29_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:08,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/e705f9374c4440639f74f02876afe158, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:08,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/e705f9374c4440639f74f02876afe158 is 175, key is test_row_0/A:col10/1731708846222/Put/seqid=0 2024-11-15T22:14:08,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742423_1599 (size=31105) 2024-11-15T22:14:08,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:08,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708908516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:08,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:08,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708908516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:08,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:08,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708908527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:08,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:08,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708908527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:08,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:08,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708908527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:08,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:08,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:08,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T22:14:08,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708908727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:08,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:08,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708908729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:08,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:08,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708908736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:08,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:08,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:08,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708908736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:08,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:08,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708908736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:08,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:08,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:08,855 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/fd58df7155e84f2bb080addc2578867e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/fd58df7155e84f2bb080addc2578867e 2024-11-15T22:14:08,901 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28230b65c0b592a41b759dc13b865232/A of 28230b65c0b592a41b759dc13b865232 into fd58df7155e84f2bb080addc2578867e(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:08,901 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:08,901 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/A, priority=13, startTime=1731708847923; duration=0sec 2024-11-15T22:14:08,902 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:08,902 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:A 2024-11-15T22:14:08,921 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/e705f9374c4440639f74f02876afe158 2024-11-15T22:14:08,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/1dd14e6f72f3415d8e8703368b6ae153 is 50, key is test_row_0/B:col10/1731708846222/Put/seqid=0 2024-11-15T22:14:09,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742424_1600 (size=12151) 2024-11-15T22:14:09,006 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/1dd14e6f72f3415d8e8703368b6ae153 2024-11-15T22:14:09,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/7ffa6c54a9e34c3d835db3d5946d5350 is 50, key is test_row_0/C:col10/1731708846222/Put/seqid=0 2024-11-15T22:14:09,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742425_1601 (size=12151) 2024-11-15T22:14:09,023 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/7ffa6c54a9e34c3d835db3d5946d5350 2024-11-15T22:14:09,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/e705f9374c4440639f74f02876afe158 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/e705f9374c4440639f74f02876afe158 2024-11-15T22:14:09,032 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/e705f9374c4440639f74f02876afe158, entries=150, sequenceid=156, filesize=30.4 K 2024-11-15T22:14:09,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/1dd14e6f72f3415d8e8703368b6ae153 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/1dd14e6f72f3415d8e8703368b6ae153 2024-11-15T22:14:09,038 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/1dd14e6f72f3415d8e8703368b6ae153, entries=150, sequenceid=156, filesize=11.9 K 2024-11-15T22:14:09,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708909036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:09,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:09,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/7ffa6c54a9e34c3d835db3d5946d5350 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/7ffa6c54a9e34c3d835db3d5946d5350 2024-11-15T22:14:09,044 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/7ffa6c54a9e34c3d835db3d5946d5350, entries=150, sequenceid=156, filesize=11.9 K 2024-11-15T22:14:09,045 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 28230b65c0b592a41b759dc13b865232 in 1012ms, sequenceid=156, compaction requested=false 2024-11-15T22:14:09,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:09,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:09,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-15T22:14:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-15T22:14:09,048 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-15T22:14:09,049 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5660 sec 2024-11-15T22:14:09,052 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.5710 sec 2024-11-15T22:14:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:09,053 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:14:09,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T22:14:09,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157fc3b5f021874c8bb1ae7d12e0973510_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708848411/Put/seqid=0 2024-11-15T22:14:09,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742426_1602 (size=12304) 2024-11-15T22:14:09,099 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:09,103 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157fc3b5f021874c8bb1ae7d12e0973510_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157fc3b5f021874c8bb1ae7d12e0973510_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:09,103 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/152734de7d494eb8b10f5a268d3eca98, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:09,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/152734de7d494eb8b10f5a268d3eca98 is 175, key is test_row_0/A:col10/1731708848411/Put/seqid=0 2024-11-15T22:14:09,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708909103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:09,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708909104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:09,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708909105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:09,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:09,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708909109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:09,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:09,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:09,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742427_1603 (size=31105) 2024-11-15T22:14:09,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708909216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:09,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:09,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708909216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:09,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:09,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708909217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:09,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708909227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:09,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:09,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708909427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:09,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:09,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708909427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:09,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:09,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708909427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:09,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,431 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:09,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708909436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:09,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:09,526 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/152734de7d494eb8b10f5a268d3eca98 2024-11-15T22:14:09,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/6874da188b674da8bdf2200c751aab1d is 50, key is test_row_0/B:col10/1731708848411/Put/seqid=0 2024-11-15T22:14:09,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742428_1604 (size=12151) 2024-11-15T22:14:09,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708909547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:09,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:09,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708909737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:09,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708909737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:09,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708909737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,741 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:09,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708909746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:09,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/6874da188b674da8bdf2200c751aab1d 2024-11-15T22:14:09,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/a9d51169988a454b81f1b4164e1ff1d8 is 50, key is test_row_0/C:col10/1731708848411/Put/seqid=0 2024-11-15T22:14:09,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742429_1605 (size=12151) 2024-11-15T22:14:09,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/a9d51169988a454b81f1b4164e1ff1d8 2024-11-15T22:14:09,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/152734de7d494eb8b10f5a268d3eca98 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/152734de7d494eb8b10f5a268d3eca98 2024-11-15T22:14:09,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/152734de7d494eb8b10f5a268d3eca98, entries=150, sequenceid=180, filesize=30.4 K 2024-11-15T22:14:09,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/6874da188b674da8bdf2200c751aab1d as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/6874da188b674da8bdf2200c751aab1d 2024-11-15T22:14:09,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/6874da188b674da8bdf2200c751aab1d, entries=150, sequenceid=180, filesize=11.9 K 2024-11-15T22:14:09,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/a9d51169988a454b81f1b4164e1ff1d8 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/a9d51169988a454b81f1b4164e1ff1d8 2024-11-15T22:14:09,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/a9d51169988a454b81f1b4164e1ff1d8, entries=150, sequenceid=180, filesize=11.9 K 2024-11-15T22:14:09,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 28230b65c0b592a41b759dc13b865232 in 929ms, sequenceid=180, compaction requested=true 2024-11-15T22:14:09,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:09,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:14:09,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:09,983 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:14:09,983 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:14:09,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:14:09,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:09,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:14:09,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:14:09,984 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:14:09,984 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/B is initiating minor compaction (all files) 2024-11-15T22:14:09,984 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/B in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:09,984 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/74334bfa276843e884d74ee9109f629f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/1dd14e6f72f3415d8e8703368b6ae153, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/6874da188b674da8bdf2200c751aab1d] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=35.9 K 2024-11-15T22:14:09,984 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:14:09,984 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/A is initiating minor compaction (all files) 2024-11-15T22:14:09,984 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/A in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:09,984 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/fd58df7155e84f2bb080addc2578867e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/e705f9374c4440639f74f02876afe158, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/152734de7d494eb8b10f5a268d3eca98] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=91.4 K 2024-11-15T22:14:09,984 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:09,984 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/fd58df7155e84f2bb080addc2578867e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/e705f9374c4440639f74f02876afe158, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/152734de7d494eb8b10f5a268d3eca98] 2024-11-15T22:14:09,985 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 74334bfa276843e884d74ee9109f629f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731708846140 2024-11-15T22:14:09,985 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dd14e6f72f3415d8e8703368b6ae153, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731708846211 2024-11-15T22:14:09,985 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd58df7155e84f2bb080addc2578867e, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731708846140 2024-11-15T22:14:09,985 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 6874da188b674da8bdf2200c751aab1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731708848411 2024-11-15T22:14:09,985 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting e705f9374c4440639f74f02876afe158, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731708846211 2024-11-15T22:14:09,986 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 152734de7d494eb8b10f5a268d3eca98, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731708848411 2024-11-15T22:14:09,999 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#B#compaction#500 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:10,000 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/2c8030c1fd494fc682ad772ebcbc132f is 50, key is test_row_0/B:col10/1731708848411/Put/seqid=0 2024-11-15T22:14:10,000 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:10,003 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115a69db088f4794e11864376f6ae4b9a5c_28230b65c0b592a41b759dc13b865232 store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:10,004 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115a69db088f4794e11864376f6ae4b9a5c_28230b65c0b592a41b759dc13b865232, store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:10,004 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a69db088f4794e11864376f6ae4b9a5c_28230b65c0b592a41b759dc13b865232 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:10,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742430_1606 (size=12561) 2024-11-15T22:14:10,032 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/2c8030c1fd494fc682ad772ebcbc132f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/2c8030c1fd494fc682ad772ebcbc132f 2024-11-15T22:14:10,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742431_1607 (size=4469) 2024-11-15T22:14:10,037 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#A#compaction#501 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:10,038 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/d1149fe747234a9f95b8cb05e5c4d795 is 175, key is test_row_0/A:col10/1731708848411/Put/seqid=0 2024-11-15T22:14:10,039 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28230b65c0b592a41b759dc13b865232/B of 28230b65c0b592a41b759dc13b865232 into 2c8030c1fd494fc682ad772ebcbc132f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:10,039 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:10,039 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/B, priority=13, startTime=1731708849983; duration=0sec 2024-11-15T22:14:10,039 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:14:10,039 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:B 2024-11-15T22:14:10,039 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:14:10,041 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:14:10,041 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/C is initiating minor compaction (all files) 2024-11-15T22:14:10,041 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/C in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:10,042 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/938fb2a9c49f4436a7ecdc25a55183c4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/7ffa6c54a9e34c3d835db3d5946d5350, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/a9d51169988a454b81f1b4164e1ff1d8] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=35.9 K 2024-11-15T22:14:10,042 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 938fb2a9c49f4436a7ecdc25a55183c4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731708846140 2024-11-15T22:14:10,042 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ffa6c54a9e34c3d835db3d5946d5350, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731708846211 2024-11-15T22:14:10,043 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting a9d51169988a454b81f1b4164e1ff1d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731708848411 2024-11-15T22:14:10,054 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#C#compaction#502 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:10,054 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/65ef5270e53f4749b6e98e581f24ca25 is 50, key is test_row_0/C:col10/1731708848411/Put/seqid=0 2024-11-15T22:14:10,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742432_1608 (size=31515) 2024-11-15T22:14:10,063 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/d1149fe747234a9f95b8cb05e5c4d795 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/d1149fe747234a9f95b8cb05e5c4d795 2024-11-15T22:14:10,068 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28230b65c0b592a41b759dc13b865232/A of 28230b65c0b592a41b759dc13b865232 into d1149fe747234a9f95b8cb05e5c4d795(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:10,068 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:10,068 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/A, priority=13, startTime=1731708849983; duration=0sec 2024-11-15T22:14:10,068 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:10,068 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:A 2024-11-15T22:14:10,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742433_1609 (size=12561) 2024-11-15T22:14:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:10,249 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:14:10,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T22:14:10,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153dbb42034a454eb7bccdd8632b3bb551_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708849100/Put/seqid=0 2024-11-15T22:14:10,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742434_1610 (size=12304) 2024-11-15T22:14:10,265 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:10,269 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153dbb42034a454eb7bccdd8632b3bb551_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153dbb42034a454eb7bccdd8632b3bb551_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:10,270 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/9672ed33d81949be9146d1a29bbeee35, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:10,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/9672ed33d81949be9146d1a29bbeee35 is 175, key is test_row_0/A:col10/1731708849100/Put/seqid=0 2024-11-15T22:14:10,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742435_1611 (size=31105) 2024-11-15T22:14:10,275 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/9672ed33d81949be9146d1a29bbeee35 2024-11-15T22:14:10,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:10,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708910272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708910273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:10,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:10,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/5f4eefe7ef774ac299cb05c514b03d6c is 50, key is test_row_0/B:col10/1731708849100/Put/seqid=0 2024-11-15T22:14:10,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742436_1612 (size=12151) 2024-11-15T22:14:10,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:10,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708910279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708910279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,286 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:10,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708910386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:10,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708910386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:10,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708910397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:10,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:10,401 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,401 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708910397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:10,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:10,482 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/65ef5270e53f4749b6e98e581f24ca25 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/65ef5270e53f4749b6e98e581f24ca25 2024-11-15T22:14:10,486 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28230b65c0b592a41b759dc13b865232/C of 28230b65c0b592a41b759dc13b865232 into 65ef5270e53f4749b6e98e581f24ca25(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:10,486 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:10,486 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/C, priority=13, startTime=1731708849983; duration=0sec 2024-11-15T22:14:10,486 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:10,486 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:C 2024-11-15T22:14:10,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:10,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708910566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:10,569 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_1' in 28230b65c0b592a41b759dc13b865232 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:14:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:10,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708910596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:10,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708910597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:10,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708910607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708910607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T22:14:10,637 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:14:10,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:14:10,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-15T22:14:10,639 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:14:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T22:14:10,640 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:14:10,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:14:10,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/5f4eefe7ef774ac299cb05c514b03d6c 2024-11-15T22:14:10,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/07531450513c4711a9eb1486d53fc349 is 50, key is test_row_0/C:col10/1731708849100/Put/seqid=0 2024-11-15T22:14:10,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742437_1613 (size=12151) 2024-11-15T22:14:10,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T22:14:10,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T22:14:10,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:10,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:10,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:10,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:10,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:10,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:10,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708910906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:10,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708910907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:10,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:10,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708910917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:10,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:10,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708910917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:10,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:10,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:10,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:10,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T22:14:10,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:10,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:10,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:10,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:10,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T22:14:11,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/07531450513c4711a9eb1486d53fc349 2024-11-15T22:14:11,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T22:14:11,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:11,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:11,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:11,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:11,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:11,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/9672ed33d81949be9146d1a29bbeee35 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/9672ed33d81949be9146d1a29bbeee35 2024-11-15T22:14:11,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/9672ed33d81949be9146d1a29bbeee35, entries=150, sequenceid=199, filesize=30.4 K 2024-11-15T22:14:11,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/5f4eefe7ef774ac299cb05c514b03d6c as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/5f4eefe7ef774ac299cb05c514b03d6c 2024-11-15T22:14:11,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/5f4eefe7ef774ac299cb05c514b03d6c, entries=150, sequenceid=199, filesize=11.9 K 2024-11-15T22:14:11,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/07531450513c4711a9eb1486d53fc349 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/07531450513c4711a9eb1486d53fc349 2024-11-15T22:14:11,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/07531450513c4711a9eb1486d53fc349, entries=150, sequenceid=199, filesize=11.9 K 2024-11-15T22:14:11,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 28230b65c0b592a41b759dc13b865232 in 880ms, sequenceid=199, compaction requested=false 2024-11-15T22:14:11,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:11,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T22:14:11,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:11,263 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T22:14:11,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T22:14:11,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ed2956b10a73466db0193b30efd6b5dc_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708850267/Put/seqid=0 2024-11-15T22:14:11,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742438_1614 (size=12304) 2024-11-15T22:14:11,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:11,302 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ed2956b10a73466db0193b30efd6b5dc_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ed2956b10a73466db0193b30efd6b5dc_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:11,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/e13d7175487549f5b1c473766d100894, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:11,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/e13d7175487549f5b1c473766d100894 is 175, key is test_row_0/A:col10/1731708850267/Put/seqid=0 2024-11-15T22:14:11,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742439_1615 (size=31105) 2024-11-15T22:14:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:11,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:11,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708911460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:11,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708911459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:11,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:11,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708911470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:11,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:11,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708911473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:11,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:11,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708911576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:11,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708911577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:11,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:11,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708911587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:11,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:11,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708911587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:11,736 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=219, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/e13d7175487549f5b1c473766d100894 2024-11-15T22:14:11,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/4a9afc0f974a4579a9cc2b0b422827aa is 50, key is test_row_0/B:col10/1731708850267/Put/seqid=0 2024-11-15T22:14:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T22:14:11,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708911786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:11,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708911787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:11,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:11,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708911797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:11,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708911797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:11,801 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:11,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742440_1616 (size=12151) 2024-11-15T22:14:11,804 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/4a9afc0f974a4579a9cc2b0b422827aa 2024-11-15T22:14:11,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/cbef5f08468041ed95999278ce7e5282 is 50, key is test_row_0/C:col10/1731708850267/Put/seqid=0 2024-11-15T22:14:11,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742441_1617 (size=12151) 2024-11-15T22:14:11,873 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/cbef5f08468041ed95999278ce7e5282 2024-11-15T22:14:11,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/e13d7175487549f5b1c473766d100894 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/e13d7175487549f5b1c473766d100894 2024-11-15T22:14:11,933 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/e13d7175487549f5b1c473766d100894, entries=150, sequenceid=219, filesize=30.4 K 2024-11-15T22:14:11,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/4a9afc0f974a4579a9cc2b0b422827aa as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/4a9afc0f974a4579a9cc2b0b422827aa 2024-11-15T22:14:11,953 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/4a9afc0f974a4579a9cc2b0b422827aa, entries=150, sequenceid=219, filesize=11.9 K 2024-11-15T22:14:11,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/cbef5f08468041ed95999278ce7e5282 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/cbef5f08468041ed95999278ce7e5282 2024-11-15T22:14:11,963 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/cbef5f08468041ed95999278ce7e5282, entries=150, sequenceid=219, filesize=11.9 K 2024-11-15T22:14:11,965 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 28230b65c0b592a41b759dc13b865232 in 701ms, sequenceid=219, compaction requested=true 2024-11-15T22:14:11,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:11,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:11,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-15T22:14:11,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-15T22:14:11,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-15T22:14:11,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3270 sec 2024-11-15T22:14:11,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.3340 sec 2024-11-15T22:14:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:12,102 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:14:12,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-15T22:14:12,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b2db311533e1436691aced39d67ad990_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708852099/Put/seqid=0 2024-11-15T22:14:12,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:12,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708912131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:12,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:12,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742442_1618 (size=17284) 2024-11-15T22:14:12,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708912133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:12,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:12,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708912139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:12,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708912140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:12,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:12,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:12,150 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:12,157 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b2db311533e1436691aced39d67ad990_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b2db311533e1436691aced39d67ad990_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:12,161 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/efd2ca2bb5ce4be4a83356963767d45f, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:12,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/efd2ca2bb5ce4be4a83356963767d45f is 175, key is test_row_0/A:col10/1731708852099/Put/seqid=0 2024-11-15T22:14:12,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742443_1619 (size=48389) 2024-11-15T22:14:12,188 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/efd2ca2bb5ce4be4a83356963767d45f 2024-11-15T22:14:12,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/8c7bd20d2192486a969f0df33c44d32e is 50, key is test_row_0/B:col10/1731708852099/Put/seqid=0 2024-11-15T22:14:12,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742444_1620 (size=12151) 2024-11-15T22:14:12,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/8c7bd20d2192486a969f0df33c44d32e 2024-11-15T22:14:12,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/bfe868accf8640da9595fb71d1296608 is 50, key is test_row_0/C:col10/1731708852099/Put/seqid=0 2024-11-15T22:14:12,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742445_1621 (size=12151) 2024-11-15T22:14:12,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:12,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708912247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:12,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:12,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708912247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:12,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:12,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708912247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:12,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:12,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708912247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:12,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708912457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:12,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:12,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708912457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:12,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708912457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:12,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:12,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708912457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:12,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:12,465 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,465 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,465 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:12,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708912586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:12,590 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_1' in 28230b65c0b592a41b759dc13b865232 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:14:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:12,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/bfe868accf8640da9595fb71d1296608 2024-11-15T22:14:12,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/efd2ca2bb5ce4be4a83356963767d45f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/efd2ca2bb5ce4be4a83356963767d45f 2024-11-15T22:14:12,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/efd2ca2bb5ce4be4a83356963767d45f, entries=250, sequenceid=239, filesize=47.3 K 2024-11-15T22:14:12,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/8c7bd20d2192486a969f0df33c44d32e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/8c7bd20d2192486a969f0df33c44d32e 2024-11-15T22:14:12,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/8c7bd20d2192486a969f0df33c44d32e, entries=150, sequenceid=239, filesize=11.9 K 2024-11-15T22:14:12,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/bfe868accf8640da9595fb71d1296608 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/bfe868accf8640da9595fb71d1296608 2024-11-15T22:14:12,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/bfe868accf8640da9595fb71d1296608, entries=150, sequenceid=239, filesize=11.9 K 2024-11-15T22:14:12,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 28230b65c0b592a41b759dc13b865232 in 639ms, sequenceid=239, compaction requested=true 2024-11-15T22:14:12,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:12,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:14:12,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:14:12,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:14:12,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T22:14:12,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:14:12,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T22:14:12,742 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:14:12,742 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:14:12,743 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:14:12,743 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/C is initiating minor compaction (all files) 2024-11-15T22:14:12,744 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/C in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:12,744 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/65ef5270e53f4749b6e98e581f24ca25, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/07531450513c4711a9eb1486d53fc349, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/cbef5f08468041ed95999278ce7e5282, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/bfe868accf8640da9595fb71d1296608] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=47.9 K 2024-11-15T22:14:12,744 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142114 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:14:12,744 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/A is initiating minor compaction (all files) 2024-11-15T22:14:12,744 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/A in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:12,744 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/d1149fe747234a9f95b8cb05e5c4d795, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/9672ed33d81949be9146d1a29bbeee35, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/e13d7175487549f5b1c473766d100894, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/efd2ca2bb5ce4be4a83356963767d45f] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=138.8 K 2024-11-15T22:14:12,744 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:12,744 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/d1149fe747234a9f95b8cb05e5c4d795, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/9672ed33d81949be9146d1a29bbeee35, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/e13d7175487549f5b1c473766d100894, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/efd2ca2bb5ce4be4a83356963767d45f] 2024-11-15T22:14:12,744 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 65ef5270e53f4749b6e98e581f24ca25, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731708848411 2024-11-15T22:14:12,745 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1149fe747234a9f95b8cb05e5c4d795, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731708848411 2024-11-15T22:14:12,745 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9672ed33d81949be9146d1a29bbeee35, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731708849093 2024-11-15T22:14:12,746 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 07531450513c4711a9eb1486d53fc349, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731708849093 2024-11-15T22:14:12,746 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting e13d7175487549f5b1c473766d100894, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731708850267 2024-11-15T22:14:12,746 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting cbef5f08468041ed95999278ce7e5282, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731708850267 2024-11-15T22:14:12,746 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting efd2ca2bb5ce4be4a83356963767d45f, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731708851453 2024-11-15T22:14:12,747 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting bfe868accf8640da9595fb71d1296608, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731708851453 2024-11-15T22:14:12,756 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:12,759 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#C#compaction#513 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:12,759 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/e30fba52086f47cab028833683e20082 is 50, key is test_row_0/C:col10/1731708852099/Put/seqid=0 2024-11-15T22:14:12,770 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111523511372a94347b6aa7ad2cf19643663_28230b65c0b592a41b759dc13b865232 store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:12,773 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111523511372a94347b6aa7ad2cf19643663_28230b65c0b592a41b759dc13b865232, store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:12,773 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111523511372a94347b6aa7ad2cf19643663_28230b65c0b592a41b759dc13b865232 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:12,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:12,779 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:14:12,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-15T22:14:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T22:14:12,797 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:14:12,797 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:14:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-15T22:14:12,801 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:14:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T22:14:12,802 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:14:12,802 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:14:12,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f0d5a46c5bd845e6a68a54ac552c3f6b_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708852133/Put/seqid=0 2024-11-15T22:14:12,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742446_1622 (size=12697) 2024-11-15T22:14:12,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742447_1623 (size=4469) 2024-11-15T22:14:12,819 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/e30fba52086f47cab028833683e20082 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/e30fba52086f47cab028833683e20082 2024-11-15T22:14:12,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742448_1624 (size=14844) 2024-11-15T22:14:12,833 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 28230b65c0b592a41b759dc13b865232/C of 28230b65c0b592a41b759dc13b865232 into e30fba52086f47cab028833683e20082(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:12,833 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:12,833 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/C, priority=12, startTime=1731708852742; duration=0sec 2024-11-15T22:14:12,833 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:14:12,833 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:C 2024-11-15T22:14:12,833 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:14:12,834 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:14:12,834 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/B is initiating minor compaction (all files) 2024-11-15T22:14:12,834 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/B in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:12,835 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/2c8030c1fd494fc682ad772ebcbc132f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/5f4eefe7ef774ac299cb05c514b03d6c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/4a9afc0f974a4579a9cc2b0b422827aa, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/8c7bd20d2192486a969f0df33c44d32e] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=47.9 K 2024-11-15T22:14:12,835 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c8030c1fd494fc682ad772ebcbc132f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731708848411 2024-11-15T22:14:12,835 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f4eefe7ef774ac299cb05c514b03d6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731708849093 2024-11-15T22:14:12,836 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a9afc0f974a4579a9cc2b0b422827aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731708850267 2024-11-15T22:14:12,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708912830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:12,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708912839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:12,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708912841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:12,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708912842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:12,846 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c7bd20d2192486a969f0df33c44d32e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731708851453 2024-11-15T22:14:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:12,858 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#B#compaction#515 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:12,858 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/8dca0e4c253c4fae9094abee559266ea is 50, key is test_row_0/B:col10/1731708852099/Put/seqid=0 2024-11-15T22:14:12,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742449_1625 (size=12697) 2024-11-15T22:14:12,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T22:14:12,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:12,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708912946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:12,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:12,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-15T22:14:12,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:12,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:12,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:12,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:12,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:12,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:12,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708912956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:12,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:12,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708912956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:12,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708912956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:13,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-15T22:14:13,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:13,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:13,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:13,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:13,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:13,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:13,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T22:14:13,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:13,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708913156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:13,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:13,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:13,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:13,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:13,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708913166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:13,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:13,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708913167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:13,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:13,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708913167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:13,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:13,213 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#A#compaction#512 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:13,214 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/63b9a3e2f3ba4801bbf94906c2309c56 is 175, key is test_row_0/A:col10/1731708852099/Put/seqid=0 2024-11-15T22:14:13,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742450_1626 (size=31651) 2024-11-15T22:14:13,223 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/63b9a3e2f3ba4801bbf94906c2309c56 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/63b9a3e2f3ba4801bbf94906c2309c56 2024-11-15T22:14:13,227 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 28230b65c0b592a41b759dc13b865232/A of 28230b65c0b592a41b759dc13b865232 into 63b9a3e2f3ba4801bbf94906c2309c56(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:13,227 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:13,227 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/A, priority=12, startTime=1731708852741; duration=0sec 2024-11-15T22:14:13,227 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:13,228 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:A 2024-11-15T22:14:13,231 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:13,234 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f0d5a46c5bd845e6a68a54ac552c3f6b_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f0d5a46c5bd845e6a68a54ac552c3f6b_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:13,235 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/1310221015eb40aeae872e6717f20227, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:13,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/1310221015eb40aeae872e6717f20227 is 175, key is test_row_0/A:col10/1731708852133/Put/seqid=0 2024-11-15T22:14:13,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742451_1627 (size=39799) 2024-11-15T22:14:13,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-15T22:14:13,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:13,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:13,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:13,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:13,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:13,273 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/8dca0e4c253c4fae9094abee559266ea as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/8dca0e4c253c4fae9094abee559266ea 2024-11-15T22:14:13,278 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 28230b65c0b592a41b759dc13b865232/B of 28230b65c0b592a41b759dc13b865232 into 8dca0e4c253c4fae9094abee559266ea(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:13,278 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:13,278 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/B, priority=12, startTime=1731708852742; duration=0sec 2024-11-15T22:14:13,278 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:13,278 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:B 2024-11-15T22:14:13,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-15T22:14:13,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:13,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:13,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:13,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:13,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:13,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T22:14:13,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708913467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:13,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:13,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:13,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:13,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:13,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708913476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:13,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:13,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708913476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:13,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708913476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:13,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-15T22:14:13,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:13,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:13,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:13,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:13,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:13,651 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/1310221015eb40aeae872e6717f20227 2024-11-15T22:14:13,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/a098afed21004a2f9fb89d57164ee16e is 50, key is test_row_0/B:col10/1731708852133/Put/seqid=0 2024-11-15T22:14:13,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742452_1628 (size=12201) 2024-11-15T22:14:13,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/a098afed21004a2f9fb89d57164ee16e 2024-11-15T22:14:13,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/7076c1e632c74003865578cbbe4a27b6 is 50, key is test_row_0/C:col10/1731708852133/Put/seqid=0 2024-11-15T22:14:13,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742453_1629 (size=12201) 2024-11-15T22:14:13,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/7076c1e632c74003865578cbbe4a27b6 2024-11-15T22:14:13,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/1310221015eb40aeae872e6717f20227 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/1310221015eb40aeae872e6717f20227 2024-11-15T22:14:13,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/1310221015eb40aeae872e6717f20227, entries=200, sequenceid=257, filesize=38.9 K 2024-11-15T22:14:13,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/a098afed21004a2f9fb89d57164ee16e as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/a098afed21004a2f9fb89d57164ee16e 2024-11-15T22:14:13,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/a098afed21004a2f9fb89d57164ee16e, entries=150, sequenceid=257, filesize=11.9 K 2024-11-15T22:14:13,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/7076c1e632c74003865578cbbe4a27b6 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/7076c1e632c74003865578cbbe4a27b6 2024-11-15T22:14:13,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/7076c1e632c74003865578cbbe4a27b6, entries=150, sequenceid=257, filesize=11.9 K 2024-11-15T22:14:13,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=114.05 KB/116790 for 28230b65c0b592a41b759dc13b865232 in 924ms, sequenceid=257, compaction requested=false 2024-11-15T22:14:13,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:13,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-15T22:14:13,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:13,731 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-15T22:14:13,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d74d6b781a7f4823b40ec446485688a7_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708852837/Put/seqid=0 2024-11-15T22:14:13,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742454_1630 (size=12454) 2024-11-15T22:14:13,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:13,759 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d74d6b781a7f4823b40ec446485688a7_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d74d6b781a7f4823b40ec446485688a7_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:13,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/c3f2b808324d4edcb6fc56f64743a77b, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:13,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/c3f2b808324d4edcb6fc56f64743a77b is 175, key is test_row_0/A:col10/1731708852837/Put/seqid=0 2024-11-15T22:14:13,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742455_1631 (size=31255) 2024-11-15T22:14:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T22:14:13,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:13,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:14,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708913997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:14,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:14,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708914000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:14,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:14,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708914002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:14,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:14,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:14,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708914003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:14,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708914107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:14,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708914116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:14,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708914116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:14,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:14,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708914116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:14,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:14,168 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/c3f2b808324d4edcb6fc56f64743a77b 2024-11-15T22:14:14,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/dc19b935820c4054a51ab02fb331ab45 is 50, key is test_row_0/B:col10/1731708852837/Put/seqid=0 2024-11-15T22:14:14,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742456_1632 (size=12301) 2024-11-15T22:14:14,201 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/dc19b935820c4054a51ab02fb331ab45 2024-11-15T22:14:14,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/d0a2b81f02dc49a2bd173d87e0a3f176 is 50, key is test_row_0/C:col10/1731708852837/Put/seqid=0 2024-11-15T22:14:14,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742457_1633 (size=12301) 2024-11-15T22:14:14,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708914317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:14,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:14,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708914327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:14,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708914327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708914327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:14,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:14,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708914626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:14,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:14,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708914637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:14,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:14,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708914637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:14,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:14,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:14,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:14,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708914637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:14,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:14,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:14,659 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/d0a2b81f02dc49a2bd173d87e0a3f176 2024-11-15T22:14:14,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/c3f2b808324d4edcb6fc56f64743a77b as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/c3f2b808324d4edcb6fc56f64743a77b 2024-11-15T22:14:14,666 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/c3f2b808324d4edcb6fc56f64743a77b, entries=150, sequenceid=280, filesize=30.5 K 2024-11-15T22:14:14,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/dc19b935820c4054a51ab02fb331ab45 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/dc19b935820c4054a51ab02fb331ab45 2024-11-15T22:14:14,670 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/dc19b935820c4054a51ab02fb331ab45, entries=150, sequenceid=280, filesize=12.0 K 2024-11-15T22:14:14,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/d0a2b81f02dc49a2bd173d87e0a3f176 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/d0a2b81f02dc49a2bd173d87e0a3f176 2024-11-15T22:14:14,675 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/d0a2b81f02dc49a2bd173d87e0a3f176, entries=150, sequenceid=280, filesize=12.0 K 2024-11-15T22:14:14,676 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 28230b65c0b592a41b759dc13b865232 in 944ms, sequenceid=280, compaction requested=true 2024-11-15T22:14:14,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:14,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:14,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-15T22:14:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-15T22:14:14,679 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-15T22:14:14,679 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8750 sec 2024-11-15T22:14:14,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.8820 sec 2024-11-15T22:14:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T22:14:14,947 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:14:14,947 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:14:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-15T22:14:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-15T22:14:14,949 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:14:14,950 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:14:14,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:14:15,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-15T22:14:15,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-15T22:14:15,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:15,103 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T22:14:15,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159d94f8f4ffe449fabc0b14e20751a712_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708854000/Put/seqid=0 2024-11-15T22:14:15,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742458_1634 (size=12454) 2024-11-15T22:14:15,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:15,120 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159d94f8f4ffe449fabc0b14e20751a712_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159d94f8f4ffe449fabc0b14e20751a712_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:15,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/808db5ab6c164925a1e026b7f9a9022d, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:15,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/808db5ab6c164925a1e026b7f9a9022d is 175, key is test_row_0/A:col10/1731708854000/Put/seqid=0 2024-11-15T22:14:15,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742459_1635 (size=31255) 2024-11-15T22:14:15,126 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/808db5ab6c164925a1e026b7f9a9022d 2024-11-15T22:14:15,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/aee65ffa5a6c4e09b105a5ab250e9073 is 50, key is test_row_0/B:col10/1731708854000/Put/seqid=0 2024-11-15T22:14:15,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742460_1636 (size=12301) 2024-11-15T22:14:15,141 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/aee65ffa5a6c4e09b105a5ab250e9073 2024-11-15T22:14:15,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/b0f1db510401482bbb3b1555f1d53346 is 50, key is test_row_0/C:col10/1731708854000/Put/seqid=0 2024-11-15T22:14:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:15,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:15,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742461_1637 (size=12301) 2024-11-15T22:14:15,153 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/b0f1db510401482bbb3b1555f1d53346 2024-11-15T22:14:15,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/808db5ab6c164925a1e026b7f9a9022d as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/808db5ab6c164925a1e026b7f9a9022d 2024-11-15T22:14:15,161 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/808db5ab6c164925a1e026b7f9a9022d, entries=150, sequenceid=296, filesize=30.5 K 2024-11-15T22:14:15,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/aee65ffa5a6c4e09b105a5ab250e9073 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/aee65ffa5a6c4e09b105a5ab250e9073 2024-11-15T22:14:15,166 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/aee65ffa5a6c4e09b105a5ab250e9073, entries=150, sequenceid=296, filesize=12.0 K 2024-11-15T22:14:15,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/b0f1db510401482bbb3b1555f1d53346 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/b0f1db510401482bbb3b1555f1d53346 2024-11-15T22:14:15,171 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/b0f1db510401482bbb3b1555f1d53346, entries=150, sequenceid=296, filesize=12.0 K 2024-11-15T22:14:15,173 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=80.51 KB/82440 for 28230b65c0b592a41b759dc13b865232 in 70ms, sequenceid=296, compaction requested=true 2024-11-15T22:14:15,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:15,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:15,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-15T22:14:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-15T22:14:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:15,176 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:14:15,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T22:14:15,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-15T22:14:15,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 225 msec 2024-11-15T22:14:15,181 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 233 msec 2024-11-15T22:14:15,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159c6dcb40c7fe4dbbae0bc96f832926e5_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708855175/Put/seqid=0 2024-11-15T22:14:15,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742462_1638 (size=12454) 2024-11-15T22:14:15,196 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:15,200 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159c6dcb40c7fe4dbbae0bc96f832926e5_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159c6dcb40c7fe4dbbae0bc96f832926e5_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:15,201 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/881a52c6c7034b1788c8341fb4fa3ff6, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:15,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/881a52c6c7034b1788c8341fb4fa3ff6 is 175, key is test_row_0/A:col10/1731708855175/Put/seqid=0 2024-11-15T22:14:15,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742463_1639 (size=31255) 2024-11-15T22:14:15,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708915202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:15,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708915203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:15,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708915204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:15,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:15,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708915208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-15T22:14:15,267 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:14:15,267 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:14:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-15T22:14:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-15T22:14:15,270 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:14:15,270 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:14:15,271 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:14:15,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:15,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708915316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:15,320 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:15,320 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,320 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:15,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708915316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708915316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:15,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708915316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-15T22:14:15,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-15T22:14:15,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:15,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:15,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:15,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:15,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:15,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708915526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:15,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708915526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:15,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:15,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:15,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:15,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708915526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708915526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:15,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:15,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:15,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-15T22:14:15,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:15,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:15,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:15,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:15,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-15T22:14:15,606 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/881a52c6c7034b1788c8341fb4fa3ff6 2024-11-15T22:14:15,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/f4d2df1594a240bf9d6961555aae7bc5 is 50, key is test_row_0/B:col10/1731708855175/Put/seqid=0 2024-11-15T22:14:15,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742464_1640 (size=12301) 2024-11-15T22:14:15,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-15T22:14:15,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:15,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:15,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:15,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:15,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:15,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:15,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708915837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:15,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708915837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:15,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708915837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:15,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708915837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:15,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:15,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-15T22:14:15,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:15,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:15,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:15,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:15,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:15,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-15T22:14:16,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/f4d2df1594a240bf9d6961555aae7bc5 2024-11-15T22:14:16,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/2a0f954ce8c8401c8949b7038b07b468 is 50, key is test_row_0/C:col10/1731708855175/Put/seqid=0 2024-11-15T22:14:16,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-15T22:14:16,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:16,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:16,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:16,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:16,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:16,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742465_1641 (size=12301) 2024-11-15T22:14:16,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/2a0f954ce8c8401c8949b7038b07b468 2024-11-15T22:14:16,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/881a52c6c7034b1788c8341fb4fa3ff6 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/881a52c6c7034b1788c8341fb4fa3ff6 2024-11-15T22:14:16,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/881a52c6c7034b1788c8341fb4fa3ff6, entries=150, sequenceid=313, filesize=30.5 K 2024-11-15T22:14:16,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/f4d2df1594a240bf9d6961555aae7bc5 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/f4d2df1594a240bf9d6961555aae7bc5 2024-11-15T22:14:16,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/f4d2df1594a240bf9d6961555aae7bc5, entries=150, sequenceid=313, filesize=12.0 K 2024-11-15T22:14:16,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/2a0f954ce8c8401c8949b7038b07b468 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/2a0f954ce8c8401c8949b7038b07b468 2024-11-15T22:14:16,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/2a0f954ce8c8401c8949b7038b07b468, entries=150, sequenceid=313, filesize=12.0 K 2024-11-15T22:14:16,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 28230b65c0b592a41b759dc13b865232 in 895ms, sequenceid=313, compaction requested=true 2024-11-15T22:14:16,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:16,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:14:16,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:16,072 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T22:14:16,072 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T22:14:16,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:14:16,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:16,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:14:16,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:14:16,073 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 165215 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T22:14:16,073 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/A is initiating minor compaction (all files) 2024-11-15T22:14:16,074 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/A in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:16,074 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/63b9a3e2f3ba4801bbf94906c2309c56, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/1310221015eb40aeae872e6717f20227, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/c3f2b808324d4edcb6fc56f64743a77b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/808db5ab6c164925a1e026b7f9a9022d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/881a52c6c7034b1788c8341fb4fa3ff6] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=161.3 K 2024-11-15T22:14:16,074 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:16,074 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/63b9a3e2f3ba4801bbf94906c2309c56, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/1310221015eb40aeae872e6717f20227, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/c3f2b808324d4edcb6fc56f64743a77b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/808db5ab6c164925a1e026b7f9a9022d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/881a52c6c7034b1788c8341fb4fa3ff6] 2024-11-15T22:14:16,074 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61801 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T22:14:16,074 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63b9a3e2f3ba4801bbf94906c2309c56, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731708851453 2024-11-15T22:14:16,074 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/B is initiating minor compaction (all files) 2024-11-15T22:14:16,074 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/B in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:16,074 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/8dca0e4c253c4fae9094abee559266ea, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/a098afed21004a2f9fb89d57164ee16e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/dc19b935820c4054a51ab02fb331ab45, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/aee65ffa5a6c4e09b105a5ab250e9073, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/f4d2df1594a240bf9d6961555aae7bc5] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=60.4 K 2024-11-15T22:14:16,075 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1310221015eb40aeae872e6717f20227, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731708852120 2024-11-15T22:14:16,075 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dca0e4c253c4fae9094abee559266ea, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731708851453 2024-11-15T22:14:16,075 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3f2b808324d4edcb6fc56f64743a77b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731708852810 2024-11-15T22:14:16,075 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting a098afed21004a2f9fb89d57164ee16e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731708852120 2024-11-15T22:14:16,075 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 808db5ab6c164925a1e026b7f9a9022d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731708853999 2024-11-15T22:14:16,076 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting dc19b935820c4054a51ab02fb331ab45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731708852810 2024-11-15T22:14:16,076 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting aee65ffa5a6c4e09b105a5ab250e9073, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731708853999 2024-11-15T22:14:16,076 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 881a52c6c7034b1788c8341fb4fa3ff6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731708855150 2024-11-15T22:14:16,076 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting f4d2df1594a240bf9d6961555aae7bc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731708855150 2024-11-15T22:14:16,090 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#B#compaction#527 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:16,091 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/ae5467cad526431f99922618885bd511 is 50, key is test_row_0/B:col10/1731708855175/Put/seqid=0 2024-11-15T22:14:16,102 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:16,110 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115d02048963ccc47359a331c9aa7f2e624_28230b65c0b592a41b759dc13b865232 store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:16,113 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115d02048963ccc47359a331c9aa7f2e624_28230b65c0b592a41b759dc13b865232, store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:16,114 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d02048963ccc47359a331c9aa7f2e624_28230b65c0b592a41b759dc13b865232 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:16,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742466_1642 (size=13017) 2024-11-15T22:14:16,126 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/ae5467cad526431f99922618885bd511 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/ae5467cad526431f99922618885bd511 2024-11-15T22:14:16,133 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 28230b65c0b592a41b759dc13b865232/B of 28230b65c0b592a41b759dc13b865232 into ae5467cad526431f99922618885bd511(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:16,133 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:16,133 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/B, priority=11, startTime=1731708856072; duration=0sec 2024-11-15T22:14:16,133 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:14:16,133 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:B 2024-11-15T22:14:16,133 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T22:14:16,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742467_1643 (size=4469) 2024-11-15T22:14:16,137 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61801 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T22:14:16,137 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/C is initiating minor compaction (all files) 2024-11-15T22:14:16,137 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/C in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:16,137 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/e30fba52086f47cab028833683e20082, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/7076c1e632c74003865578cbbe4a27b6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/d0a2b81f02dc49a2bd173d87e0a3f176, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/b0f1db510401482bbb3b1555f1d53346, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/2a0f954ce8c8401c8949b7038b07b468] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=60.4 K 2024-11-15T22:14:16,142 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting e30fba52086f47cab028833683e20082, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731708851453 2024-11-15T22:14:16,143 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 7076c1e632c74003865578cbbe4a27b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731708852120 2024-11-15T22:14:16,143 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#A#compaction#528 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:16,143 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting d0a2b81f02dc49a2bd173d87e0a3f176, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731708852810 2024-11-15T22:14:16,143 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/2d786d2be9f4459eb736b28bafd94ff4 is 175, key is test_row_0/A:col10/1731708855175/Put/seqid=0 2024-11-15T22:14:16,143 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting b0f1db510401482bbb3b1555f1d53346, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731708853999 2024-11-15T22:14:16,144 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a0f954ce8c8401c8949b7038b07b468, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731708855150 2024-11-15T22:14:16,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742468_1644 (size=31971) 2024-11-15T22:14:16,169 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/2d786d2be9f4459eb736b28bafd94ff4 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/2d786d2be9f4459eb736b28bafd94ff4 2024-11-15T22:14:16,175 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 28230b65c0b592a41b759dc13b865232/A of 28230b65c0b592a41b759dc13b865232 into 2d786d2be9f4459eb736b28bafd94ff4(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:16,175 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:16,175 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/A, priority=11, startTime=1731708856072; duration=0sec 2024-11-15T22:14:16,175 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:16,175 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:A 2024-11-15T22:14:16,184 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#C#compaction#529 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:16,185 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/28dd3df680a64845856975d48c3d28f3 is 50, key is test_row_0/C:col10/1731708855175/Put/seqid=0 2024-11-15T22:14:16,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-15T22:14:16,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:16,190 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T22:14:16,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742469_1645 (size=13017) 2024-11-15T22:14:16,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153a7453b183f74fd9bfe4cc7dda32ff8d_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708855200/Put/seqid=0 2024-11-15T22:14:16,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742470_1646 (size=12454) 2024-11-15T22:14:16,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:16,213 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153a7453b183f74fd9bfe4cc7dda32ff8d_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153a7453b183f74fd9bfe4cc7dda32ff8d_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:16,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/0f55966552b4409d93b3544fe2eda6ab, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:16,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/0f55966552b4409d93b3544fe2eda6ab is 175, key is test_row_0/A:col10/1731708855200/Put/seqid=0 2024-11-15T22:14:16,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742471_1647 (size=31255) 2024-11-15T22:14:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:16,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:16,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708916374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:16,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708916374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:16,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:16,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:16,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708916375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:16,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708916377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:16,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-15T22:14:16,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708916487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:16,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708916487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:16,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:16,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708916506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:16,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708916506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:16,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:16,599 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/28dd3df680a64845856975d48c3d28f3 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/28dd3df680a64845856975d48c3d28f3 2024-11-15T22:14:16,603 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 28230b65c0b592a41b759dc13b865232/C of 28230b65c0b592a41b759dc13b865232 into 28dd3df680a64845856975d48c3d28f3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:16,603 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:16,604 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/C, priority=11, startTime=1731708856072; duration=0sec 2024-11-15T22:14:16,604 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:16,604 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:C 2024-11-15T22:14:16,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48514 deadline: 1731708916626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:16,631 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_1' in 28230b65c0b592a41b759dc13b865232 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:14:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:16,641 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/0f55966552b4409d93b3544fe2eda6ab 2024-11-15T22:14:16,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/503c729e542a405499dee8b2dd6926c7 is 50, key is test_row_0/B:col10/1731708855200/Put/seqid=0 2024-11-15T22:14:16,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742472_1648 (size=12301) 2024-11-15T22:14:16,660 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/503c729e542a405499dee8b2dd6926c7 2024-11-15T22:14:16,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/3474408c19294664bc428f7f66873b32 is 50, key is test_row_0/C:col10/1731708855200/Put/seqid=0 2024-11-15T22:14:16,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742473_1649 (size=12301) 2024-11-15T22:14:16,679 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/3474408c19294664bc428f7f66873b32 2024-11-15T22:14:16,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/0f55966552b4409d93b3544fe2eda6ab as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/0f55966552b4409d93b3544fe2eda6ab 2024-11-15T22:14:16,689 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/0f55966552b4409d93b3544fe2eda6ab, entries=150, sequenceid=334, filesize=30.5 K 2024-11-15T22:14:16,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/503c729e542a405499dee8b2dd6926c7 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/503c729e542a405499dee8b2dd6926c7 2024-11-15T22:14:16,697 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/503c729e542a405499dee8b2dd6926c7, entries=150, sequenceid=334, filesize=12.0 K 2024-11-15T22:14:16,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/3474408c19294664bc428f7f66873b32 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/3474408c19294664bc428f7f66873b32 2024-11-15T22:14:16,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:16,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708916696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:16,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:16,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708916697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:16,702 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/3474408c19294664bc428f7f66873b32, entries=150, sequenceid=334, filesize=12.0 K 2024-11-15T22:14:16,703 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 28230b65c0b592a41b759dc13b865232 in 513ms, sequenceid=334, compaction requested=false 2024-11-15T22:14:16,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:16,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:16,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-15T22:14:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-15T22:14:16,707 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-15T22:14:16,707 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4340 sec 2024-11-15T22:14:16,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.4400 sec 2024-11-15T22:14:16,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:16,719 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:14:16,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-15T22:14:16,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115676fc6b0f3874146bf95f5867053944e_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708856367/Put/seqid=0 2024-11-15T22:14:16,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:16,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708916767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:16,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:16,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:16,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708916768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:16,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:16,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742474_1650 (size=14994) 2024-11-15T22:14:16,780 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:16,785 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115676fc6b0f3874146bf95f5867053944e_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115676fc6b0f3874146bf95f5867053944e_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:16,786 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/774be67803c248c2b8cb25b59ef99872, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:16,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/774be67803c248c2b8cb25b59ef99872 is 175, key is test_row_0/A:col10/1731708856367/Put/seqid=0 2024-11-15T22:14:16,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742475_1651 (size=39949) 2024-11-15T22:14:16,812 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/774be67803c248c2b8cb25b59ef99872 2024-11-15T22:14:16,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/14033efee3dd43d8ab37541b14c7ae9a is 50, key is test_row_0/B:col10/1731708856367/Put/seqid=0 2024-11-15T22:14:16,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742476_1652 (size=12301) 2024-11-15T22:14:16,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/14033efee3dd43d8ab37541b14c7ae9a 2024-11-15T22:14:16,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/f880b6e2718a4c8da9cb51fd4f52c375 is 50, key is test_row_0/C:col10/1731708856367/Put/seqid=0 2024-11-15T22:14:16,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742477_1653 (size=12301) 2024-11-15T22:14:16,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708916876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:16,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708916876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:16,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:17,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708917007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:17,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708917007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:17,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708917087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:17,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:17,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708917087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:17,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:17,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/f880b6e2718a4c8da9cb51fd4f52c375 2024-11-15T22:14:17,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/774be67803c248c2b8cb25b59ef99872 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/774be67803c248c2b8cb25b59ef99872 2024-11-15T22:14:17,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/774be67803c248c2b8cb25b59ef99872, entries=200, sequenceid=353, filesize=39.0 K 2024-11-15T22:14:17,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/14033efee3dd43d8ab37541b14c7ae9a as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/14033efee3dd43d8ab37541b14c7ae9a 2024-11-15T22:14:17,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/14033efee3dd43d8ab37541b14c7ae9a, entries=150, sequenceid=353, filesize=12.0 K 2024-11-15T22:14:17,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/f880b6e2718a4c8da9cb51fd4f52c375 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/f880b6e2718a4c8da9cb51fd4f52c375 2024-11-15T22:14:17,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/f880b6e2718a4c8da9cb51fd4f52c375, entries=150, sequenceid=353, filesize=12.0 K 2024-11-15T22:14:17,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 28230b65c0b592a41b759dc13b865232 in 554ms, sequenceid=353, compaction requested=true 2024-11-15T22:14:17,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:17,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:14:17,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:17,274 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:14:17,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:14:17,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:17,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:14:17,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:14:17,274 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:14:17,275 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:14:17,275 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:14:17,275 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/A is initiating minor compaction (all files) 2024-11-15T22:14:17,275 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/B is initiating minor compaction (all files) 2024-11-15T22:14:17,275 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/A in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:17,275 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/B in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:17,275 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/2d786d2be9f4459eb736b28bafd94ff4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/0f55966552b4409d93b3544fe2eda6ab, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/774be67803c248c2b8cb25b59ef99872] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=100.8 K 2024-11-15T22:14:17,275 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/ae5467cad526431f99922618885bd511, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/503c729e542a405499dee8b2dd6926c7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/14033efee3dd43d8ab37541b14c7ae9a] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=36.7 K 2024-11-15T22:14:17,275 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:17,275 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/2d786d2be9f4459eb736b28bafd94ff4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/0f55966552b4409d93b3544fe2eda6ab, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/774be67803c248c2b8cb25b59ef99872] 2024-11-15T22:14:17,275 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d786d2be9f4459eb736b28bafd94ff4, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731708855150 2024-11-15T22:14:17,275 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting ae5467cad526431f99922618885bd511, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731708855150 2024-11-15T22:14:17,276 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 503c729e542a405499dee8b2dd6926c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731708855200 2024-11-15T22:14:17,276 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f55966552b4409d93b3544fe2eda6ab, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731708855200 2024-11-15T22:14:17,276 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 774be67803c248c2b8cb25b59ef99872, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731708856361 2024-11-15T22:14:17,276 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 14033efee3dd43d8ab37541b14c7ae9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731708856361 2024-11-15T22:14:17,281 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:17,283 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115268aceef761f4ba1988eeacfc0639b82_28230b65c0b592a41b759dc13b865232 store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:17,284 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#B#compaction#537 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:17,285 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/0829e181fcbf490c87a9740f3ae326ed is 50, key is test_row_0/B:col10/1731708856367/Put/seqid=0 2024-11-15T22:14:17,285 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115268aceef761f4ba1988eeacfc0639b82_28230b65c0b592a41b759dc13b865232, store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:17,285 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115268aceef761f4ba1988eeacfc0639b82_28230b65c0b592a41b759dc13b865232 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:17,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742479_1655 (size=13119) 2024-11-15T22:14:17,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742478_1654 (size=4469) 2024-11-15T22:14:17,298 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#A#compaction#536 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:17,298 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/e0564b99d85442f3860e646163699f39 is 175, key is test_row_0/A:col10/1731708856367/Put/seqid=0 2024-11-15T22:14:17,304 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/0829e181fcbf490c87a9740f3ae326ed as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/0829e181fcbf490c87a9740f3ae326ed 2024-11-15T22:14:17,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742480_1656 (size=32073) 2024-11-15T22:14:17,309 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28230b65c0b592a41b759dc13b865232/B of 28230b65c0b592a41b759dc13b865232 into 0829e181fcbf490c87a9740f3ae326ed(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:17,309 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:17,309 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/B, priority=13, startTime=1731708857274; duration=0sec 2024-11-15T22:14:17,309 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:14:17,309 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:B 2024-11-15T22:14:17,310 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:14:17,310 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:14:17,310 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/C is initiating minor compaction (all files) 2024-11-15T22:14:17,310 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/C in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:17,311 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/28dd3df680a64845856975d48c3d28f3, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/3474408c19294664bc428f7f66873b32, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/f880b6e2718a4c8da9cb51fd4f52c375] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=36.7 K 2024-11-15T22:14:17,311 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 28dd3df680a64845856975d48c3d28f3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731708855150 2024-11-15T22:14:17,311 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 3474408c19294664bc428f7f66873b32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731708855200 2024-11-15T22:14:17,311 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting f880b6e2718a4c8da9cb51fd4f52c375, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731708856361 2024-11-15T22:14:17,317 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#C#compaction#538 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:17,317 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/8a17162ad17f4c939c023e63b22b8cbc is 50, key is test_row_0/C:col10/1731708856367/Put/seqid=0 2024-11-15T22:14:17,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742481_1657 (size=13119) 2024-11-15T22:14:17,335 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/8a17162ad17f4c939c023e63b22b8cbc as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/8a17162ad17f4c939c023e63b22b8cbc 2024-11-15T22:14:17,349 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28230b65c0b592a41b759dc13b865232/C of 28230b65c0b592a41b759dc13b865232 into 8a17162ad17f4c939c023e63b22b8cbc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:17,350 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:17,350 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/C, priority=13, startTime=1731708857274; duration=0sec 2024-11-15T22:14:17,350 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:17,350 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:C 2024-11-15T22:14:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:17,399 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:14:17,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T22:14:17,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111594f69ed53f8a441480250cf1c26fab0b_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708857399/Put/seqid=0 2024-11-15T22:14:17,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742482_1658 (size=14994) 2024-11-15T22:14:17,412 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:17,416 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111594f69ed53f8a441480250cf1c26fab0b_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111594f69ed53f8a441480250cf1c26fab0b_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-15T22:14:17,417 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:14:17,417 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/eb900363758c418fb9b09b8664b8788c, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:17,417 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T22:14:17,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/eb900363758c418fb9b09b8664b8788c is 175, key is test_row_0/A:col10/1731708857399/Put/seqid=0 2024-11-15T22:14:17,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-15T22:14:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-15T22:14:17,419 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T22:14:17,420 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T22:14:17,420 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T22:14:17,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742483_1659 (size=39949) 2024-11-15T22:14:17,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:17,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708917435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:17,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708917436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:17,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:17,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708917516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:17,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:17,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708917517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:17,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-15T22:14:17,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:17,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708917546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:17,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708917546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:17,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-15T22:14:17,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:17,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:17,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:17,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:17,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:17,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:17,709 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/e0564b99d85442f3860e646163699f39 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/e0564b99d85442f3860e646163699f39 2024-11-15T22:14:17,713 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28230b65c0b592a41b759dc13b865232/A of 28230b65c0b592a41b759dc13b865232 into e0564b99d85442f3860e646163699f39(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:17,713 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:17,714 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/A, priority=13, startTime=1731708857274; duration=0sec 2024-11-15T22:14:17,714 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:17,714 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:A 2024-11-15T22:14:17,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-15T22:14:17,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:17,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:17,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:17,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:17,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:17,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-15T22:14:17,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:17,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708917756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:17,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708917756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:17,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:17,830 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=375, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/eb900363758c418fb9b09b8664b8788c 2024-11-15T22:14:17,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/14d6b4c3f53e442b8cc24724a1e959fb is 50, key is test_row_0/B:col10/1731708857399/Put/seqid=0 2024-11-15T22:14:17,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742484_1660 (size=12301) 2024-11-15T22:14:17,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-15T22:14:17,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:17,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:17,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:17,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:17,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:18,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-15T22:14:18,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:18,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:18,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:18,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:18,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-15T22:14:18,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708918067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:18,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708918067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:18,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-15T22:14:18,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:18,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:18,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:18,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:18,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:18,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/14d6b4c3f53e442b8cc24724a1e959fb 2024-11-15T22:14:18,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/9d6a2bb8a2a242a5bc45f71c687bb74a is 50, key is test_row_0/C:col10/1731708857399/Put/seqid=0 2024-11-15T22:14:18,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742485_1661 (size=12301) 2024-11-15T22:14:18,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/9d6a2bb8a2a242a5bc45f71c687bb74a 2024-11-15T22:14:18,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/eb900363758c418fb9b09b8664b8788c as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/eb900363758c418fb9b09b8664b8788c 2024-11-15T22:14:18,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/eb900363758c418fb9b09b8664b8788c, entries=200, sequenceid=375, filesize=39.0 K 2024-11-15T22:14:18,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/14d6b4c3f53e442b8cc24724a1e959fb as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/14d6b4c3f53e442b8cc24724a1e959fb 2024-11-15T22:14:18,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/14d6b4c3f53e442b8cc24724a1e959fb, entries=150, sequenceid=375, filesize=12.0 K 2024-11-15T22:14:18,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/9d6a2bb8a2a242a5bc45f71c687bb74a as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/9d6a2bb8a2a242a5bc45f71c687bb74a 2024-11-15T22:14:18,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/9d6a2bb8a2a242a5bc45f71c687bb74a, entries=150, sequenceid=375, filesize=12.0 K 2024-11-15T22:14:18,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 28230b65c0b592a41b759dc13b865232 in 883ms, sequenceid=375, compaction requested=false 2024-11-15T22:14:18,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:18,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37917 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-15T22:14:18,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:18,339 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T22:14:18,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153cfbf9a14ce6400fac169a8bb234a304_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708857427/Put/seqid=0 2024-11-15T22:14:18,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742486_1662 (size=12454) 2024-11-15T22:14:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:18,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. as already flushing 2024-11-15T22:14:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-15T22:14:18,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708918574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:18,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:18,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708918578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:18,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:18,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:18,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:18,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708918578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:18,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:18,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:18,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708918586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:18,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708918687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:18,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708918696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:18,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:18,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:18,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:18,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:18,763 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153cfbf9a14ce6400fac169a8bb234a304_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153cfbf9a14ce6400fac169a8bb234a304_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:18,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/4b8114404e35433d8c2e451cc889c8bb, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:18,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/4b8114404e35433d8c2e451cc889c8bb is 175, key is test_row_0/A:col10/1731708857427/Put/seqid=0 2024-11-15T22:14:18,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742487_1663 (size=31255) 2024-11-15T22:14:18,771 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=392, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/4b8114404e35433d8c2e451cc889c8bb 2024-11-15T22:14:18,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/b85464a30dbb43a3a8ecd36991b647c3 is 50, key is test_row_0/B:col10/1731708857427/Put/seqid=0 2024-11-15T22:14:18,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742488_1664 (size=12301) 2024-11-15T22:14:18,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708918897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:18,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708918906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:18,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:18,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:18,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:19,183 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/b85464a30dbb43a3a8ecd36991b647c3 2024-11-15T22:14:19,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/0a4851c734b2462bbd1dcf4f31c46f29 is 50, key is test_row_0/C:col10/1731708857427/Put/seqid=0 2024-11-15T22:14:19,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742489_1665 (size=12301) 2024-11-15T22:14:19,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708919212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:19,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:19,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:19,215 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:19,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708919217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:19,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:19,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:19,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-15T22:14:19,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708919596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:19,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708919596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:19,600 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_2' in 28230b65c0b592a41b759dc13b865232 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:14:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:19,601 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_0' in 28230b65c0b592a41b759dc13b865232 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:14:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:19,602 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/0a4851c734b2462bbd1dcf4f31c46f29 2024-11-15T22:14:19,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/4b8114404e35433d8c2e451cc889c8bb as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/4b8114404e35433d8c2e451cc889c8bb 2024-11-15T22:14:19,615 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/4b8114404e35433d8c2e451cc889c8bb, entries=150, sequenceid=392, filesize=30.5 K 2024-11-15T22:14:19,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/b85464a30dbb43a3a8ecd36991b647c3 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/b85464a30dbb43a3a8ecd36991b647c3 2024-11-15T22:14:19,620 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/b85464a30dbb43a3a8ecd36991b647c3, entries=150, sequenceid=392, filesize=12.0 K 2024-11-15T22:14:19,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/0a4851c734b2462bbd1dcf4f31c46f29 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/0a4851c734b2462bbd1dcf4f31c46f29 2024-11-15T22:14:19,625 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/0a4851c734b2462bbd1dcf4f31c46f29, entries=150, sequenceid=392, filesize=12.0 K 2024-11-15T22:14:19,626 INFO [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 28230b65c0b592a41b759dc13b865232 in 1288ms, sequenceid=392, compaction requested=true 2024-11-15T22:14:19,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:19,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:19,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-15T22:14:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-15T22:14:19,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-15T22:14:19,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2080 sec 2024-11-15T22:14:19,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 2.2120 sec 2024-11-15T22:14:19,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:19,729 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:14:19,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T22:14:19,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154b706dff664a4901a159a03ce8754926_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708858571/Put/seqid=0 2024-11-15T22:14:19,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742490_1666 (size=14994) 2024-11-15T22:14:19,743 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:19,747 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154b706dff664a4901a159a03ce8754926_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154b706dff664a4901a159a03ce8754926_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:19,748 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/81be4c5f033041e9a4e524a68c9bd183, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:19,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/81be4c5f033041e9a4e524a68c9bd183 is 175, key is test_row_0/A:col10/1731708858571/Put/seqid=0 2024-11-15T22:14:19,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742491_1667 (size=39949) 2024-11-15T22:14:19,752 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=413, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/81be4c5f033041e9a4e524a68c9bd183 2024-11-15T22:14:19,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/33d23547957141f69a0e0d2c1ca700e2 is 50, key is test_row_0/B:col10/1731708858571/Put/seqid=0 2024-11-15T22:14:19,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:19,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708919763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:19,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:19,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:19,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:19,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:19,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742492_1668 (size=12301) 2024-11-15T22:14:19,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708919769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:19,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:19,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:19,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:19,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708919877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:19,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708919877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:19,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:19,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:19,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:19,883 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:19,883 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:19,883 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:20,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:20,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708920086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:20,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:20,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708920087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:20,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/33d23547957141f69a0e0d2c1ca700e2 2024-11-15T22:14:20,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/78d95bc69b0c4692bf90a9bd70045e61 is 50, key is test_row_0/C:col10/1731708858571/Put/seqid=0 2024-11-15T22:14:20,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742493_1669 (size=12301) 2024-11-15T22:14:20,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708920397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:20,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708920397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:20,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/78d95bc69b0c4692bf90a9bd70045e61 2024-11-15T22:14:20,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/81be4c5f033041e9a4e524a68c9bd183 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/81be4c5f033041e9a4e524a68c9bd183 2024-11-15T22:14:20,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/81be4c5f033041e9a4e524a68c9bd183, entries=200, sequenceid=413, filesize=39.0 K 2024-11-15T22:14:20,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/33d23547957141f69a0e0d2c1ca700e2 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/33d23547957141f69a0e0d2c1ca700e2 2024-11-15T22:14:20,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/33d23547957141f69a0e0d2c1ca700e2, entries=150, sequenceid=413, filesize=12.0 K 2024-11-15T22:14:20,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/78d95bc69b0c4692bf90a9bd70045e61 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/78d95bc69b0c4692bf90a9bd70045e61 2024-11-15T22:14:20,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/78d95bc69b0c4692bf90a9bd70045e61, entries=150, sequenceid=413, filesize=12.0 K 2024-11-15T22:14:20,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 28230b65c0b592a41b759dc13b865232 in 876ms, sequenceid=413, compaction requested=true 2024-11-15T22:14:20,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:20,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:14:20,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:20,606 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:14:20,606 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:14:20,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:14:20,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:20,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:14:20,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:14:20,614 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:14:20,614 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/A is initiating minor compaction (all files) 2024-11-15T22:14:20,614 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/A in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:20,614 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/e0564b99d85442f3860e646163699f39, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/eb900363758c418fb9b09b8664b8788c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/4b8114404e35433d8c2e451cc889c8bb, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/81be4c5f033041e9a4e524a68c9bd183] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=139.9 K 2024-11-15T22:14:20,614 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:20,614 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/e0564b99d85442f3860e646163699f39, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/eb900363758c418fb9b09b8664b8788c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/4b8114404e35433d8c2e451cc889c8bb, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/81be4c5f033041e9a4e524a68c9bd183] 2024-11-15T22:14:20,615 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0564b99d85442f3860e646163699f39, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731708856361 2024-11-15T22:14:20,615 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:14:20,615 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/B is initiating minor compaction (all files) 2024-11-15T22:14:20,615 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/B in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:20,615 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/0829e181fcbf490c87a9740f3ae326ed, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/14d6b4c3f53e442b8cc24724a1e959fb, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/b85464a30dbb43a3a8ecd36991b647c3, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/33d23547957141f69a0e0d2c1ca700e2] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=48.8 K 2024-11-15T22:14:20,615 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb900363758c418fb9b09b8664b8788c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731708856765 2024-11-15T22:14:20,616 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b8114404e35433d8c2e451cc889c8bb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731708857427 2024-11-15T22:14:20,616 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 0829e181fcbf490c87a9740f3ae326ed, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731708856361 2024-11-15T22:14:20,617 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81be4c5f033041e9a4e524a68c9bd183, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731708858558 2024-11-15T22:14:20,617 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 14d6b4c3f53e442b8cc24724a1e959fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731708856765 2024-11-15T22:14:20,617 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting b85464a30dbb43a3a8ecd36991b647c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731708857427 2024-11-15T22:14:20,617 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 33d23547957141f69a0e0d2c1ca700e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731708858558 2024-11-15T22:14:20,625 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:20,627 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115cbd10de98778448fae4cf40115428792_28230b65c0b592a41b759dc13b865232 store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:20,629 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#B#compaction#549 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:20,630 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115cbd10de98778448fae4cf40115428792_28230b65c0b592a41b759dc13b865232, store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:20,630 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cbd10de98778448fae4cf40115428792_28230b65c0b592a41b759dc13b865232 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:20,630 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/8abaefc1b4d443379298ff1214fb72e4 is 50, key is test_row_0/B:col10/1731708858571/Put/seqid=0 2024-11-15T22:14:20,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742495_1671 (size=13255) 2024-11-15T22:14:20,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742494_1670 (size=4469) 2024-11-15T22:14:20,647 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#A#compaction#548 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:20,648 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/a20ab6df0d7d45559e634616cd17e57a is 175, key is test_row_0/A:col10/1731708858571/Put/seqid=0 2024-11-15T22:14:20,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742496_1672 (size=32209) 2024-11-15T22:14:20,666 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/a20ab6df0d7d45559e634616cd17e57a as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/a20ab6df0d7d45559e634616cd17e57a 2024-11-15T22:14:20,671 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 28230b65c0b592a41b759dc13b865232/A of 28230b65c0b592a41b759dc13b865232 into a20ab6df0d7d45559e634616cd17e57a(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:20,671 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:20,671 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/A, priority=12, startTime=1731708860605; duration=0sec 2024-11-15T22:14:20,671 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:14:20,671 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:A 2024-11-15T22:14:20,672 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T22:14:20,674 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T22:14:20,674 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/C is initiating minor compaction (all files) 2024-11-15T22:14:20,674 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/C in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:20,674 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/8a17162ad17f4c939c023e63b22b8cbc, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/9d6a2bb8a2a242a5bc45f71c687bb74a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/0a4851c734b2462bbd1dcf4f31c46f29, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/78d95bc69b0c4692bf90a9bd70045e61] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=48.8 K 2024-11-15T22:14:20,674 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a17162ad17f4c939c023e63b22b8cbc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731708856361 2024-11-15T22:14:20,675 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d6a2bb8a2a242a5bc45f71c687bb74a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731708856765 2024-11-15T22:14:20,675 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a4851c734b2462bbd1dcf4f31c46f29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731708857427 2024-11-15T22:14:20,675 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78d95bc69b0c4692bf90a9bd70045e61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731708858558 2024-11-15T22:14:20,685 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#C#compaction#550 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:20,686 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/8d69ee336a6e4b53b098cf05795ea10a is 50, key is test_row_0/C:col10/1731708858571/Put/seqid=0 2024-11-15T22:14:20,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742497_1673 (size=13255) 2024-11-15T22:14:20,698 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/8d69ee336a6e4b53b098cf05795ea10a as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/8d69ee336a6e4b53b098cf05795ea10a 2024-11-15T22:14:20,703 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 28230b65c0b592a41b759dc13b865232/C of 28230b65c0b592a41b759dc13b865232 into 8d69ee336a6e4b53b098cf05795ea10a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:20,703 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:20,703 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/C, priority=12, startTime=1731708860606; duration=0sec 2024-11-15T22:14:20,703 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:20,703 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:C 2024-11-15T22:14:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:20,909 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:14:20,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T22:14:20,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115053181a62f2742c29182b16cf6a253e7_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708859761/Put/seqid=0 2024-11-15T22:14:20,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742498_1674 (size=12454) 2024-11-15T22:14:20,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708920954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:20,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708920955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:20,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:20,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:20,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:20,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:21,043 INFO [Thread-2491 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2491. 2024-11-15T22:14:21,043 DEBUG [Thread-2491 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:14:21,043 DEBUG [Thread-2491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:21,044 INFO [Thread-2495 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2495. 2024-11-15T22:14:21,044 DEBUG [Thread-2495 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:14:21,044 DEBUG [Thread-2495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:21,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:14:21,045 DEBUG [Thread-2495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:21,047 INFO [Thread-2493 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2493. 2024-11-15T22:14:21,047 DEBUG [Thread-2493 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:14:21,047 DEBUG [Thread-2493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:21,047 DEBUG [Thread-2493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:21,047 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:14:21,047 INFO [Thread-2489 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2489. 2024-11-15T22:14:21,047 DEBUG [Thread-2489 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:14:21,047 DEBUG [Thread-2489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:21,048 DEBUG [Thread-2489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:21,048 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:14:21,048 INFO [Thread-2497 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2497. 2024-11-15T22:14:21,048 DEBUG [Thread-2497 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:14:21,048 DEBUG [Thread-2497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:21,049 DEBUG [Thread-2497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:21,049 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:14:21,049 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:14:21,049 DEBUG [Thread-2491 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:21,052 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/8abaefc1b4d443379298ff1214fb72e4 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/8abaefc1b4d443379298ff1214fb72e4 2024-11-15T22:14:21,056 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 28230b65c0b592a41b759dc13b865232/B of 28230b65c0b592a41b759dc13b865232 into 8abaefc1b4d443379298ff1214fb72e4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:21,056 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:21,056 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/B, priority=12, startTime=1731708860606; duration=0sec 2024-11-15T22:14:21,056 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:21,056 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:B 2024-11-15T22:14:21,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:21,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708921066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708921066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:21,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:21,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:21,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:21,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:21,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:21,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:21,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:21,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:21,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708921277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:21,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708921277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:21,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:21,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:21,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:21,278 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:21,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:21,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:21,331 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:21,341 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115053181a62f2742c29182b16cf6a253e7_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115053181a62f2742c29182b16cf6a253e7_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:21,341 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/64214e9201fd46caaa8553c9a2b25fc5, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:21,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/64214e9201fd46caaa8553c9a2b25fc5 is 175, key is test_row_0/A:col10/1731708859761/Put/seqid=0 2024-11-15T22:14:21,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742499_1675 (size=31255) 2024-11-15T22:14:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-15T22:14:21,577 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:14:21,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:21,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708921586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708921586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:21,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48558 deadline: 1731708921617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:21,618 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_0' in 28230b65c0b592a41b759dc13b865232 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:14:21,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:21,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:21,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:21,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:21,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48540 deadline: 1731708921626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:21,627 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to ae3311ea5ccb,37917,1731708686516 for 'test_row_2' in 28230b65c0b592a41b759dc13b865232 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T22:14:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:21,746 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=431, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/64214e9201fd46caaa8553c9a2b25fc5 2024-11-15T22:14:21,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/df49dcd8195346008d7c603e485524bd is 50, key is test_row_0/B:col10/1731708859761/Put/seqid=0 2024-11-15T22:14:21,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742500_1676 (size=12301) 2024-11-15T22:14:21,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/df49dcd8195346008d7c603e485524bd 2024-11-15T22:14:21,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/9d074dbe700c4b8ab3785b15379b6986 is 50, key is test_row_0/C:col10/1731708859761/Put/seqid=0 2024-11-15T22:14:21,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742501_1677 (size=12301) 2024-11-15T22:14:22,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:22,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T22:14:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48530 deadline: 1731708922097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48578 deadline: 1731708922096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 , the old value is region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T22:14:22,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., hostname=ae3311ea5ccb,37917,1731708686516, seqNum=5 because the exception is null or not the one we care about 2024-11-15T22:14:22,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/9d074dbe700c4b8ab3785b15379b6986 2024-11-15T22:14:22,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/64214e9201fd46caaa8553c9a2b25fc5 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/64214e9201fd46caaa8553c9a2b25fc5 2024-11-15T22:14:22,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/64214e9201fd46caaa8553c9a2b25fc5, entries=150, sequenceid=431, filesize=30.5 K 2024-11-15T22:14:22,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/df49dcd8195346008d7c603e485524bd as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/df49dcd8195346008d7c603e485524bd 2024-11-15T22:14:22,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/df49dcd8195346008d7c603e485524bd, entries=150, sequenceid=431, filesize=12.0 K 2024-11-15T22:14:22,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/9d074dbe700c4b8ab3785b15379b6986 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/9d074dbe700c4b8ab3785b15379b6986 2024-11-15T22:14:22,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/9d074dbe700c4b8ab3785b15379b6986, entries=150, sequenceid=431, filesize=12.0 K 2024-11-15T22:14:22,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 28230b65c0b592a41b759dc13b865232 in 1270ms, sequenceid=431, compaction requested=false 2024-11-15T22:14:22,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37917 {}] regionserver.HRegion(8855): Flush requested on 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:23,108 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T22:14:23,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T22:14:23,108 INFO [Thread-2484 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2484. 2024-11-15T22:14:23,109 DEBUG [Thread-2484 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:14:23,109 DEBUG [Thread-2484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:23,109 DEBUG [Thread-2484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:23,109 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:14:23,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e8216468fdfc4eb38f06a7af812c952b_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708863107/Put/seqid=0 2024-11-15T22:14:23,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742502_1678 (size=12454) 2024-11-15T22:14:23,118 INFO [Thread-2478 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2478. 2024-11-15T22:14:23,118 DEBUG [Thread-2478 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:14:23,118 DEBUG [Thread-2478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:23,118 DEBUG [Thread-2478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:23,118 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:14:23,517 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:23,519 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e8216468fdfc4eb38f06a7af812c952b_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e8216468fdfc4eb38f06a7af812c952b_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:23,520 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/5478a525d1dd4743aaa43a865f0827c9, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:23,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/5478a525d1dd4743aaa43a865f0827c9 is 175, key is test_row_0/A:col10/1731708863107/Put/seqid=0 2024-11-15T22:14:23,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742503_1679 (size=31255) 2024-11-15T22:14:23,924 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=453, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/5478a525d1dd4743aaa43a865f0827c9 2024-11-15T22:14:23,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/954100a220ce4bdc87ceb295c9846c23 is 50, key is test_row_0/B:col10/1731708863107/Put/seqid=0 2024-11-15T22:14:23,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742504_1680 (size=12301) 2024-11-15T22:14:24,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/954100a220ce4bdc87ceb295c9846c23 2024-11-15T22:14:24,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/fd6cd89233a64972958fb33d8d3699db is 50, key is test_row_0/C:col10/1731708863107/Put/seqid=0 2024-11-15T22:14:24,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742505_1681 (size=12301) 2024-11-15T22:14:24,575 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T22:14:24,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/fd6cd89233a64972958fb33d8d3699db 2024-11-15T22:14:24,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/5478a525d1dd4743aaa43a865f0827c9 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/5478a525d1dd4743aaa43a865f0827c9 2024-11-15T22:14:24,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/5478a525d1dd4743aaa43a865f0827c9, entries=150, sequenceid=453, filesize=30.5 K 2024-11-15T22:14:24,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/954100a220ce4bdc87ceb295c9846c23 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/954100a220ce4bdc87ceb295c9846c23 2024-11-15T22:14:24,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/954100a220ce4bdc87ceb295c9846c23, entries=150, sequenceid=453, filesize=12.0 K 2024-11-15T22:14:24,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/fd6cd89233a64972958fb33d8d3699db as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/fd6cd89233a64972958fb33d8d3699db 2024-11-15T22:14:24,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/fd6cd89233a64972958fb33d8d3699db, entries=150, sequenceid=453, filesize=12.0 K 2024-11-15T22:14:24,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=6.71 KB/6870 for 28230b65c0b592a41b759dc13b865232 in 1646ms, sequenceid=453, compaction requested=true 2024-11-15T22:14:24,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:24,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T22:14:24,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:24,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T22:14:24,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:24,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 28230b65c0b592a41b759dc13b865232:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T22:14:24,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:14:24,755 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:14:24,755 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:14:24,755 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94719 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:14:24,755 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:14:24,755 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/A is initiating minor compaction (all files) 2024-11-15T22:14:24,755 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/B is initiating minor compaction (all files) 2024-11-15T22:14:24,755 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/A in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:24,755 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/B in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:24,756 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/a20ab6df0d7d45559e634616cd17e57a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/64214e9201fd46caaa8553c9a2b25fc5, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/5478a525d1dd4743aaa43a865f0827c9] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=92.5 K 2024-11-15T22:14:24,756 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/8abaefc1b4d443379298ff1214fb72e4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/df49dcd8195346008d7c603e485524bd, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/954100a220ce4bdc87ceb295c9846c23] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=37.0 K 2024-11-15T22:14:24,756 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:24,756 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. files: [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/a20ab6df0d7d45559e634616cd17e57a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/64214e9201fd46caaa8553c9a2b25fc5, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/5478a525d1dd4743aaa43a865f0827c9] 2024-11-15T22:14:24,756 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 8abaefc1b4d443379298ff1214fb72e4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731708858558 2024-11-15T22:14:24,756 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting a20ab6df0d7d45559e634616cd17e57a, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731708858558 2024-11-15T22:14:24,756 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting df49dcd8195346008d7c603e485524bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731708859760 2024-11-15T22:14:24,756 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64214e9201fd46caaa8553c9a2b25fc5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731708859760 2024-11-15T22:14:24,756 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5478a525d1dd4743aaa43a865f0827c9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731708860946 2024-11-15T22:14:24,756 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 954100a220ce4bdc87ceb295c9846c23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731708860946 2024-11-15T22:14:24,762 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:24,763 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#B#compaction#557 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:24,763 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/e0720baf01854a5f9f6cd8a0976eacbd is 50, key is test_row_0/B:col10/1731708863107/Put/seqid=0 2024-11-15T22:14:24,764 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111523f01b094bc944c1929f6a886455cba9_28230b65c0b592a41b759dc13b865232 store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:24,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742506_1682 (size=13357) 2024-11-15T22:14:24,767 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111523f01b094bc944c1929f6a886455cba9_28230b65c0b592a41b759dc13b865232, store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:24,767 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111523f01b094bc944c1929f6a886455cba9_28230b65c0b592a41b759dc13b865232 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:24,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742507_1683 (size=4469) 2024-11-15T22:14:24,770 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/e0720baf01854a5f9f6cd8a0976eacbd as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/e0720baf01854a5f9f6cd8a0976eacbd 2024-11-15T22:14:24,774 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28230b65c0b592a41b759dc13b865232/B of 28230b65c0b592a41b759dc13b865232 into e0720baf01854a5f9f6cd8a0976eacbd(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:24,774 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:24,774 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/B, priority=13, startTime=1731708864755; duration=0sec 2024-11-15T22:14:24,774 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T22:14:24,774 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:B 2024-11-15T22:14:24,774 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T22:14:24,775 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T22:14:24,775 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1541): 28230b65c0b592a41b759dc13b865232/C is initiating minor compaction (all files) 2024-11-15T22:14:24,775 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 28230b65c0b592a41b759dc13b865232/C in TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:24,775 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/8d69ee336a6e4b53b098cf05795ea10a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/9d074dbe700c4b8ab3785b15379b6986, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/fd6cd89233a64972958fb33d8d3699db] into tmpdir=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp, totalSize=37.0 K 2024-11-15T22:14:24,775 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d69ee336a6e4b53b098cf05795ea10a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731708858558 2024-11-15T22:14:24,775 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d074dbe700c4b8ab3785b15379b6986, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731708859760 2024-11-15T22:14:24,775 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] compactions.Compactor(225): Compacting fd6cd89233a64972958fb33d8d3699db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731708860946 2024-11-15T22:14:24,780 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#C#compaction#559 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:24,781 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/d13f82b975aa4384ad745fba937f9431 is 50, key is test_row_0/C:col10/1731708863107/Put/seqid=0 2024-11-15T22:14:24,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742508_1684 (size=13357) 2024-11-15T22:14:25,171 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 28230b65c0b592a41b759dc13b865232#A#compaction#558 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T22:14:25,172 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/fffe5ad793dd4b319156bd2cc232ad69 is 175, key is test_row_0/A:col10/1731708863107/Put/seqid=0 2024-11-15T22:14:25,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742509_1685 (size=32311) 2024-11-15T22:14:25,187 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/d13f82b975aa4384ad745fba937f9431 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/d13f82b975aa4384ad745fba937f9431 2024-11-15T22:14:25,191 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28230b65c0b592a41b759dc13b865232/C of 28230b65c0b592a41b759dc13b865232 into d13f82b975aa4384ad745fba937f9431(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:25,191 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:25,191 INFO [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/C, priority=13, startTime=1731708864755; duration=0sec 2024-11-15T22:14:25,191 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:25,191 DEBUG [RS:0;ae3311ea5ccb:37917-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:C 2024-11-15T22:14:25,579 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/fffe5ad793dd4b319156bd2cc232ad69 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/fffe5ad793dd4b319156bd2cc232ad69 2024-11-15T22:14:25,582 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 28230b65c0b592a41b759dc13b865232/A of 28230b65c0b592a41b759dc13b865232 into fffe5ad793dd4b319156bd2cc232ad69(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T22:14:25,582 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 28230b65c0b592a41b759dc13b865232: 2024-11-15T22:14:25,582 INFO [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., storeName=28230b65c0b592a41b759dc13b865232/A, priority=13, startTime=1731708864755; duration=0sec 2024-11-15T22:14:25,582 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T22:14:25,582 DEBUG [RS:0;ae3311ea5ccb:37917-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 28230b65c0b592a41b759dc13b865232:A 2024-11-15T22:14:25,648 INFO [Thread-2480 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2480. 2024-11-15T22:14:25,648 DEBUG [Thread-2480 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:14:25,648 DEBUG [Thread-2480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:25,648 DEBUG [Thread-2480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:25,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:14:25,658 INFO [Thread-2482 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2482. 2024-11-15T22:14:25,658 DEBUG [Thread-2482 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:14:25,658 DEBUG [Thread-2482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:25,658 DEBUG [Thread-2482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:25,658 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:14:26,648 INFO [Thread-2486 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2486. 2024-11-15T22:14:26,648 DEBUG [Thread-2486 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T22:14:26,648 DEBUG [Thread-2486 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:26,648 DEBUG [Thread-2486 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:26,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:14:26,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T22:14:26,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-15T22:14:26,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-15T22:14:26,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-15T22:14:26,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-15T22:14:26,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-15T22:14:26,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T22:14:26,649 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2786 2024-11-15T22:14:26,649 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2883 2024-11-15T22:14:26,649 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2723 2024-11-15T22:14:26,649 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2802 2024-11-15T22:14:26,649 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2872 2024-11-15T22:14:26,649 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T22:14:26,649 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T22:14:26,649 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:14:26,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:26,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:26,649 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:14:26,650 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T22:14:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T22:14:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-15T22:14:26,653 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708866652"}]},"ts":"1731708866652"} 2024-11-15T22:14:26,654 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T22:14:26,654 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T22:14:26,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T22:14:26,656 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=176, ppid=175, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=28230b65c0b592a41b759dc13b865232, UNASSIGN}] 2024-11-15T22:14:26,657 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=176, ppid=175, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=28230b65c0b592a41b759dc13b865232, UNASSIGN 2024-11-15T22:14:26,657 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=176 updating hbase:meta row=28230b65c0b592a41b759dc13b865232, regionState=CLOSING, regionLocation=ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:26,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=176, ppid=175, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=28230b65c0b592a41b759dc13b865232, UNASSIGN because future has completed 2024-11-15T22:14:26,659 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T22:14:26,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE, hasLock=false; CloseRegionProcedure 28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516}] 2024-11-15T22:14:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-15T22:14:26,811 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] handler.UnassignRegionHandler(122): Close 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:26,811 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T22:14:26,811 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1722): Closing 28230b65c0b592a41b759dc13b865232, disabling compactions & flushes 2024-11-15T22:14:26,811 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:26,811 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:26,811 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. after waiting 0 ms 2024-11-15T22:14:26,811 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:26,812 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(2902): Flushing 28230b65c0b592a41b759dc13b865232 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-15T22:14:26,817 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115acbfa846a5c74cf8a5d810ff268f8231_28230b65c0b592a41b759dc13b865232 is 50, key is test_row_0/A:col10/1731708865657/Put/seqid=0 2024-11-15T22:14:26,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742510_1686 (size=12454) 2024-11-15T22:14:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-15T22:14:27,221 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:27,224 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115acbfa846a5c74cf8a5d810ff268f8231_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115acbfa846a5c74cf8a5d810ff268f8231_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:27,224 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/69c5d54641ec490e96eb53412de6b65f, store: [table=TestAcidGuarantees family=A region=28230b65c0b592a41b759dc13b865232] 2024-11-15T22:14:27,225 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/69c5d54641ec490e96eb53412de6b65f is 175, key is test_row_0/A:col10/1731708865657/Put/seqid=0 2024-11-15T22:14:27,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742511_1687 (size=31255) 2024-11-15T22:14:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-15T22:14:27,629 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=463, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/69c5d54641ec490e96eb53412de6b65f 2024-11-15T22:14:27,635 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/1436d4d5778849768299086356da122f is 50, key is test_row_0/B:col10/1731708865657/Put/seqid=0 2024-11-15T22:14:27,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742512_1688 (size=12301) 2024-11-15T22:14:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-15T22:14:28,039 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/1436d4d5778849768299086356da122f 2024-11-15T22:14:28,045 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/3bd820be359e44b7ae15bd8122c72ff0 is 50, key is test_row_0/C:col10/1731708865657/Put/seqid=0 2024-11-15T22:14:28,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742513_1689 (size=12301) 2024-11-15T22:14:28,448 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/3bd820be359e44b7ae15bd8122c72ff0 2024-11-15T22:14:28,452 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/A/69c5d54641ec490e96eb53412de6b65f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/69c5d54641ec490e96eb53412de6b65f 2024-11-15T22:14:28,456 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/69c5d54641ec490e96eb53412de6b65f, entries=150, sequenceid=463, filesize=30.5 K 2024-11-15T22:14:28,456 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/B/1436d4d5778849768299086356da122f as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/1436d4d5778849768299086356da122f 2024-11-15T22:14:28,460 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/1436d4d5778849768299086356da122f, entries=150, sequenceid=463, filesize=12.0 K 2024-11-15T22:14:28,460 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/.tmp/C/3bd820be359e44b7ae15bd8122c72ff0 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/3bd820be359e44b7ae15bd8122c72ff0 2024-11-15T22:14:28,463 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/3bd820be359e44b7ae15bd8122c72ff0, entries=150, sequenceid=463, filesize=12.0 K 2024-11-15T22:14:28,464 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 28230b65c0b592a41b759dc13b865232 in 1653ms, sequenceid=463, compaction requested=false 2024-11-15T22:14:28,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/f75b6aa148df4df58ad0eb64608a1cf8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/828313d5abbe4977bc77112d2ba8f2e2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/573426b6b06140f4b87680440e5f3c96, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/709da9ef645b462cb52c9e9cb1a2abe9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/d656b325692749dfa185c1a3dbaaa419, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/64dfa8c256d843d79ad658a3af55aa9d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/3ca75e2e5d25469cadf28de960f34021, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/35a5ca162c1e422ab332cfc12706b4a6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/fd58df7155e84f2bb080addc2578867e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/b4e47fb9bc5841e99b7ec55d370f2c09, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/e705f9374c4440639f74f02876afe158, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/d1149fe747234a9f95b8cb05e5c4d795, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/152734de7d494eb8b10f5a268d3eca98, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/9672ed33d81949be9146d1a29bbeee35, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/e13d7175487549f5b1c473766d100894, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/efd2ca2bb5ce4be4a83356963767d45f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/63b9a3e2f3ba4801bbf94906c2309c56, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/1310221015eb40aeae872e6717f20227, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/c3f2b808324d4edcb6fc56f64743a77b, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/808db5ab6c164925a1e026b7f9a9022d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/2d786d2be9f4459eb736b28bafd94ff4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/881a52c6c7034b1788c8341fb4fa3ff6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/0f55966552b4409d93b3544fe2eda6ab, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/774be67803c248c2b8cb25b59ef99872, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/e0564b99d85442f3860e646163699f39, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/eb900363758c418fb9b09b8664b8788c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/4b8114404e35433d8c2e451cc889c8bb, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/81be4c5f033041e9a4e524a68c9bd183, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/a20ab6df0d7d45559e634616cd17e57a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/64214e9201fd46caaa8553c9a2b25fc5, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/5478a525d1dd4743aaa43a865f0827c9] to archive 2024-11-15T22:14:28,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:14:28,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/f75b6aa148df4df58ad0eb64608a1cf8 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/f75b6aa148df4df58ad0eb64608a1cf8 2024-11-15T22:14:28,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/828313d5abbe4977bc77112d2ba8f2e2 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/828313d5abbe4977bc77112d2ba8f2e2 2024-11-15T22:14:28,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/573426b6b06140f4b87680440e5f3c96 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/573426b6b06140f4b87680440e5f3c96 2024-11-15T22:14:28,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/709da9ef645b462cb52c9e9cb1a2abe9 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/709da9ef645b462cb52c9e9cb1a2abe9 2024-11-15T22:14:28,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/d656b325692749dfa185c1a3dbaaa419 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/d656b325692749dfa185c1a3dbaaa419 2024-11-15T22:14:28,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/64dfa8c256d843d79ad658a3af55aa9d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/64dfa8c256d843d79ad658a3af55aa9d 2024-11-15T22:14:28,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/3ca75e2e5d25469cadf28de960f34021 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/3ca75e2e5d25469cadf28de960f34021 2024-11-15T22:14:28,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/35a5ca162c1e422ab332cfc12706b4a6 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/35a5ca162c1e422ab332cfc12706b4a6 2024-11-15T22:14:28,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/fd58df7155e84f2bb080addc2578867e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/fd58df7155e84f2bb080addc2578867e 2024-11-15T22:14:28,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/b4e47fb9bc5841e99b7ec55d370f2c09 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/b4e47fb9bc5841e99b7ec55d370f2c09 2024-11-15T22:14:28,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/e705f9374c4440639f74f02876afe158 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/e705f9374c4440639f74f02876afe158 2024-11-15T22:14:28,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/d1149fe747234a9f95b8cb05e5c4d795 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/d1149fe747234a9f95b8cb05e5c4d795 2024-11-15T22:14:28,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/152734de7d494eb8b10f5a268d3eca98 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/152734de7d494eb8b10f5a268d3eca98 2024-11-15T22:14:28,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/9672ed33d81949be9146d1a29bbeee35 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/9672ed33d81949be9146d1a29bbeee35 2024-11-15T22:14:28,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/e13d7175487549f5b1c473766d100894 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/e13d7175487549f5b1c473766d100894 2024-11-15T22:14:28,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/efd2ca2bb5ce4be4a83356963767d45f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/efd2ca2bb5ce4be4a83356963767d45f 2024-11-15T22:14:28,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/63b9a3e2f3ba4801bbf94906c2309c56 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/63b9a3e2f3ba4801bbf94906c2309c56 2024-11-15T22:14:28,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/1310221015eb40aeae872e6717f20227 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/1310221015eb40aeae872e6717f20227 2024-11-15T22:14:28,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/c3f2b808324d4edcb6fc56f64743a77b to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/c3f2b808324d4edcb6fc56f64743a77b 2024-11-15T22:14:28,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/808db5ab6c164925a1e026b7f9a9022d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/808db5ab6c164925a1e026b7f9a9022d 2024-11-15T22:14:28,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/2d786d2be9f4459eb736b28bafd94ff4 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/2d786d2be9f4459eb736b28bafd94ff4 2024-11-15T22:14:28,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/881a52c6c7034b1788c8341fb4fa3ff6 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/881a52c6c7034b1788c8341fb4fa3ff6 2024-11-15T22:14:28,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/0f55966552b4409d93b3544fe2eda6ab to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/0f55966552b4409d93b3544fe2eda6ab 2024-11-15T22:14:28,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/774be67803c248c2b8cb25b59ef99872 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/774be67803c248c2b8cb25b59ef99872 2024-11-15T22:14:28,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/e0564b99d85442f3860e646163699f39 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/e0564b99d85442f3860e646163699f39 2024-11-15T22:14:28,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/eb900363758c418fb9b09b8664b8788c to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/eb900363758c418fb9b09b8664b8788c 2024-11-15T22:14:28,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/4b8114404e35433d8c2e451cc889c8bb to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/4b8114404e35433d8c2e451cc889c8bb 2024-11-15T22:14:28,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/81be4c5f033041e9a4e524a68c9bd183 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/81be4c5f033041e9a4e524a68c9bd183 2024-11-15T22:14:28,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/a20ab6df0d7d45559e634616cd17e57a to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/a20ab6df0d7d45559e634616cd17e57a 2024-11-15T22:14:28,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/64214e9201fd46caaa8553c9a2b25fc5 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/64214e9201fd46caaa8553c9a2b25fc5 2024-11-15T22:14:28,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/5478a525d1dd4743aaa43a865f0827c9 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/5478a525d1dd4743aaa43a865f0827c9 2024-11-15T22:14:28,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9c6967e198a14eb4abec8d77f84d17f8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9afc4bbc92b94c8282ff6282ff6a6481, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/2db242e14d2444a5b05de7a099ca14b9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/027f192a544d4dbf965082e14f020256, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9db2e21db9e24d2bae78df2d2c7602d6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/ea1105a92fb64681a9db70c1cd47b537, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9e72daf5f75a42a9b5deac0b3946a8d1, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/34d01dbfce924f23bae1f779917730f7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/74334bfa276843e884d74ee9109f629f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/23e9a5326530441e87acdffea310f3d4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/1dd14e6f72f3415d8e8703368b6ae153, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/2c8030c1fd494fc682ad772ebcbc132f, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/6874da188b674da8bdf2200c751aab1d, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/5f4eefe7ef774ac299cb05c514b03d6c, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/4a9afc0f974a4579a9cc2b0b422827aa, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/8dca0e4c253c4fae9094abee559266ea, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/8c7bd20d2192486a969f0df33c44d32e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/a098afed21004a2f9fb89d57164ee16e, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/dc19b935820c4054a51ab02fb331ab45, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/aee65ffa5a6c4e09b105a5ab250e9073, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/ae5467cad526431f99922618885bd511, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/f4d2df1594a240bf9d6961555aae7bc5, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/503c729e542a405499dee8b2dd6926c7, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/0829e181fcbf490c87a9740f3ae326ed, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/14033efee3dd43d8ab37541b14c7ae9a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/14d6b4c3f53e442b8cc24724a1e959fb, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/b85464a30dbb43a3a8ecd36991b647c3, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/8abaefc1b4d443379298ff1214fb72e4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/33d23547957141f69a0e0d2c1ca700e2, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/df49dcd8195346008d7c603e485524bd, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/954100a220ce4bdc87ceb295c9846c23] to archive 2024-11-15T22:14:28,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:14:28,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9c6967e198a14eb4abec8d77f84d17f8 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9c6967e198a14eb4abec8d77f84d17f8 2024-11-15T22:14:28,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9afc4bbc92b94c8282ff6282ff6a6481 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9afc4bbc92b94c8282ff6282ff6a6481 2024-11-15T22:14:28,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/2db242e14d2444a5b05de7a099ca14b9 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/2db242e14d2444a5b05de7a099ca14b9 2024-11-15T22:14:28,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/027f192a544d4dbf965082e14f020256 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/027f192a544d4dbf965082e14f020256 2024-11-15T22:14:28,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9db2e21db9e24d2bae78df2d2c7602d6 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9db2e21db9e24d2bae78df2d2c7602d6 2024-11-15T22:14:28,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/ea1105a92fb64681a9db70c1cd47b537 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/ea1105a92fb64681a9db70c1cd47b537 2024-11-15T22:14:28,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9e72daf5f75a42a9b5deac0b3946a8d1 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/9e72daf5f75a42a9b5deac0b3946a8d1 2024-11-15T22:14:28,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/34d01dbfce924f23bae1f779917730f7 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/34d01dbfce924f23bae1f779917730f7 2024-11-15T22:14:28,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/74334bfa276843e884d74ee9109f629f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/74334bfa276843e884d74ee9109f629f 2024-11-15T22:14:28,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/23e9a5326530441e87acdffea310f3d4 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/23e9a5326530441e87acdffea310f3d4 2024-11-15T22:14:28,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/1dd14e6f72f3415d8e8703368b6ae153 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/1dd14e6f72f3415d8e8703368b6ae153 2024-11-15T22:14:28,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/2c8030c1fd494fc682ad772ebcbc132f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/2c8030c1fd494fc682ad772ebcbc132f 2024-11-15T22:14:28,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/6874da188b674da8bdf2200c751aab1d to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/6874da188b674da8bdf2200c751aab1d 2024-11-15T22:14:28,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/5f4eefe7ef774ac299cb05c514b03d6c to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/5f4eefe7ef774ac299cb05c514b03d6c 2024-11-15T22:14:28,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/4a9afc0f974a4579a9cc2b0b422827aa to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/4a9afc0f974a4579a9cc2b0b422827aa 2024-11-15T22:14:28,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/8dca0e4c253c4fae9094abee559266ea to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/8dca0e4c253c4fae9094abee559266ea 2024-11-15T22:14:28,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/8c7bd20d2192486a969f0df33c44d32e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/8c7bd20d2192486a969f0df33c44d32e 2024-11-15T22:14:28,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/a098afed21004a2f9fb89d57164ee16e to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/a098afed21004a2f9fb89d57164ee16e 2024-11-15T22:14:28,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/dc19b935820c4054a51ab02fb331ab45 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/dc19b935820c4054a51ab02fb331ab45 2024-11-15T22:14:28,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/aee65ffa5a6c4e09b105a5ab250e9073 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/aee65ffa5a6c4e09b105a5ab250e9073 2024-11-15T22:14:28,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/ae5467cad526431f99922618885bd511 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/ae5467cad526431f99922618885bd511 2024-11-15T22:14:28,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/f4d2df1594a240bf9d6961555aae7bc5 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/f4d2df1594a240bf9d6961555aae7bc5 2024-11-15T22:14:28,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/503c729e542a405499dee8b2dd6926c7 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/503c729e542a405499dee8b2dd6926c7 2024-11-15T22:14:28,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/0829e181fcbf490c87a9740f3ae326ed to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/0829e181fcbf490c87a9740f3ae326ed 2024-11-15T22:14:28,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/14033efee3dd43d8ab37541b14c7ae9a to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/14033efee3dd43d8ab37541b14c7ae9a 2024-11-15T22:14:28,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/14d6b4c3f53e442b8cc24724a1e959fb to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/14d6b4c3f53e442b8cc24724a1e959fb 2024-11-15T22:14:28,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/b85464a30dbb43a3a8ecd36991b647c3 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/b85464a30dbb43a3a8ecd36991b647c3 2024-11-15T22:14:28,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/8abaefc1b4d443379298ff1214fb72e4 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/8abaefc1b4d443379298ff1214fb72e4 2024-11-15T22:14:28,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/33d23547957141f69a0e0d2c1ca700e2 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/33d23547957141f69a0e0d2c1ca700e2 2024-11-15T22:14:28,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/df49dcd8195346008d7c603e485524bd to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/df49dcd8195346008d7c603e485524bd 2024-11-15T22:14:28,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/954100a220ce4bdc87ceb295c9846c23 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/954100a220ce4bdc87ceb295c9846c23 2024-11-15T22:14:28,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/f8c125714cae43f3bbe31111d5193647, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/8b8986a799834c7a9c977ff0814999b9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/e8a10c72bd354098ac8afa58c2c62d60, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/7e03166b1cfd4df9994b7ce90a4b8dbb, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/4f54e8cfada947ab9f06714561aca4d9, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/cd2e56bcd9364bf0a5cefec8d317ed31, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/24c36fe63b2c44d8b0cc067d8bcde189, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/00b0cc8f106d4ffa9dbef376e4ab7869, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/938fb2a9c49f4436a7ecdc25a55183c4, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/72c83be6ba45491e93e90cc104934898, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/7ffa6c54a9e34c3d835db3d5946d5350, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/65ef5270e53f4749b6e98e581f24ca25, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/a9d51169988a454b81f1b4164e1ff1d8, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/07531450513c4711a9eb1486d53fc349, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/cbef5f08468041ed95999278ce7e5282, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/e30fba52086f47cab028833683e20082, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/bfe868accf8640da9595fb71d1296608, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/7076c1e632c74003865578cbbe4a27b6, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/d0a2b81f02dc49a2bd173d87e0a3f176, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/b0f1db510401482bbb3b1555f1d53346, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/28dd3df680a64845856975d48c3d28f3, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/2a0f954ce8c8401c8949b7038b07b468, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/3474408c19294664bc428f7f66873b32, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/8a17162ad17f4c939c023e63b22b8cbc, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/f880b6e2718a4c8da9cb51fd4f52c375, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/9d6a2bb8a2a242a5bc45f71c687bb74a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/0a4851c734b2462bbd1dcf4f31c46f29, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/8d69ee336a6e4b53b098cf05795ea10a, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/78d95bc69b0c4692bf90a9bd70045e61, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/9d074dbe700c4b8ab3785b15379b6986, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/fd6cd89233a64972958fb33d8d3699db] to archive 2024-11-15T22:14:28,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T22:14:28,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/f8c125714cae43f3bbe31111d5193647 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/f8c125714cae43f3bbe31111d5193647 2024-11-15T22:14:28,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/8b8986a799834c7a9c977ff0814999b9 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/8b8986a799834c7a9c977ff0814999b9 2024-11-15T22:14:28,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/e8a10c72bd354098ac8afa58c2c62d60 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/e8a10c72bd354098ac8afa58c2c62d60 2024-11-15T22:14:28,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/7e03166b1cfd4df9994b7ce90a4b8dbb to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/7e03166b1cfd4df9994b7ce90a4b8dbb 2024-11-15T22:14:28,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/4f54e8cfada947ab9f06714561aca4d9 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/4f54e8cfada947ab9f06714561aca4d9 2024-11-15T22:14:28,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/cd2e56bcd9364bf0a5cefec8d317ed31 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/cd2e56bcd9364bf0a5cefec8d317ed31 2024-11-15T22:14:28,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/24c36fe63b2c44d8b0cc067d8bcde189 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/24c36fe63b2c44d8b0cc067d8bcde189 2024-11-15T22:14:28,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/00b0cc8f106d4ffa9dbef376e4ab7869 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/00b0cc8f106d4ffa9dbef376e4ab7869 2024-11-15T22:14:28,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/938fb2a9c49f4436a7ecdc25a55183c4 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/938fb2a9c49f4436a7ecdc25a55183c4 2024-11-15T22:14:28,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/72c83be6ba45491e93e90cc104934898 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/72c83be6ba45491e93e90cc104934898 2024-11-15T22:14:28,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/7ffa6c54a9e34c3d835db3d5946d5350 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/7ffa6c54a9e34c3d835db3d5946d5350 2024-11-15T22:14:28,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/65ef5270e53f4749b6e98e581f24ca25 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/65ef5270e53f4749b6e98e581f24ca25 2024-11-15T22:14:28,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/a9d51169988a454b81f1b4164e1ff1d8 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/a9d51169988a454b81f1b4164e1ff1d8 2024-11-15T22:14:28,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/07531450513c4711a9eb1486d53fc349 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/07531450513c4711a9eb1486d53fc349 2024-11-15T22:14:28,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/cbef5f08468041ed95999278ce7e5282 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/cbef5f08468041ed95999278ce7e5282 2024-11-15T22:14:28,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/e30fba52086f47cab028833683e20082 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/e30fba52086f47cab028833683e20082 2024-11-15T22:14:28,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/bfe868accf8640da9595fb71d1296608 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/bfe868accf8640da9595fb71d1296608 2024-11-15T22:14:28,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/7076c1e632c74003865578cbbe4a27b6 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/7076c1e632c74003865578cbbe4a27b6 2024-11-15T22:14:28,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/d0a2b81f02dc49a2bd173d87e0a3f176 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/d0a2b81f02dc49a2bd173d87e0a3f176 2024-11-15T22:14:28,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/b0f1db510401482bbb3b1555f1d53346 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/b0f1db510401482bbb3b1555f1d53346 2024-11-15T22:14:28,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/28dd3df680a64845856975d48c3d28f3 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/28dd3df680a64845856975d48c3d28f3 2024-11-15T22:14:28,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/2a0f954ce8c8401c8949b7038b07b468 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/2a0f954ce8c8401c8949b7038b07b468 2024-11-15T22:14:28,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/3474408c19294664bc428f7f66873b32 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/3474408c19294664bc428f7f66873b32 2024-11-15T22:14:28,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/8a17162ad17f4c939c023e63b22b8cbc to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/8a17162ad17f4c939c023e63b22b8cbc 2024-11-15T22:14:28,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/f880b6e2718a4c8da9cb51fd4f52c375 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/f880b6e2718a4c8da9cb51fd4f52c375 2024-11-15T22:14:28,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/9d6a2bb8a2a242a5bc45f71c687bb74a to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/9d6a2bb8a2a242a5bc45f71c687bb74a 2024-11-15T22:14:28,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/0a4851c734b2462bbd1dcf4f31c46f29 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/0a4851c734b2462bbd1dcf4f31c46f29 2024-11-15T22:14:28,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/8d69ee336a6e4b53b098cf05795ea10a to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/8d69ee336a6e4b53b098cf05795ea10a 2024-11-15T22:14:28,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/78d95bc69b0c4692bf90a9bd70045e61 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/78d95bc69b0c4692bf90a9bd70045e61 2024-11-15T22:14:28,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/9d074dbe700c4b8ab3785b15379b6986 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/9d074dbe700c4b8ab3785b15379b6986 2024-11-15T22:14:28,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/fd6cd89233a64972958fb33d8d3699db to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/fd6cd89233a64972958fb33d8d3699db 2024-11-15T22:14:28,566 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/recovered.edits/466.seqid, newMaxSeqId=466, maxSeqId=4 2024-11-15T22:14:28,566 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. 2024-11-15T22:14:28,566 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1676): Region close journal for 28230b65c0b592a41b759dc13b865232: Waiting for close lock at 1731708866811Running coprocessor pre-close hooks at 1731708866811Disabling compacts and flushes for region at 1731708866811Disabling writes for close at 1731708866811Obtaining lock to block concurrent updates at 1731708866812 (+1 ms)Preparing flush snapshotting stores in 28230b65c0b592a41b759dc13b865232 at 1731708866812Finished memstore snapshotting TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731708866812Flushing stores of TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232. at 1731708866813 (+1 ms)Flushing 28230b65c0b592a41b759dc13b865232/A: creating writer at 1731708866813Flushing mob file 28230b65c0b592a41b759dc13b865232/A: appending metadata at 1731708866817 (+4 ms)Flushing mob file 28230b65c0b592a41b759dc13b865232/A: closing flushed file at 1731708866817Flushing 28230b65c0b592a41b759dc13b865232/A: appending metadata at 1731708867224 (+407 ms)Flushing 28230b65c0b592a41b759dc13b865232/A: closing flushed file at 1731708867224Flushing 28230b65c0b592a41b759dc13b865232/B: creating writer at 1731708867632 (+408 ms)Flushing 28230b65c0b592a41b759dc13b865232/B: appending metadata at 1731708867634 (+2 ms)Flushing 28230b65c0b592a41b759dc13b865232/B: closing flushed file at 1731708867634Flushing 28230b65c0b592a41b759dc13b865232/C: creating writer at 1731708868042 (+408 ms)Flushing 28230b65c0b592a41b759dc13b865232/C: appending metadata at 1731708868044 (+2 ms)Flushing 28230b65c0b592a41b759dc13b865232/C: closing flushed file at 1731708868045 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6749d26d: reopening flushed file at 1731708868452 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37b5043f: reopening flushed file at 1731708868456 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70b61d95: reopening flushed file at 1731708868460 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 28230b65c0b592a41b759dc13b865232 in 1653ms, sequenceid=463, compaction requested=false at 1731708868464 (+4 ms)Writing region close event to WAL at 1731708868563 (+99 ms)Running coprocessor post-close hooks at 1731708868566 (+3 ms)Closed at 1731708868566 2024-11-15T22:14:28,568 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] handler.UnassignRegionHandler(157): Closed 28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,569 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=176 updating hbase:meta row=28230b65c0b592a41b759dc13b865232, regionState=CLOSED 2024-11-15T22:14:28,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=177, ppid=176, state=RUNNABLE, hasLock=false; CloseRegionProcedure 28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 because future has completed 2024-11-15T22:14:28,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-15T22:14:28,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; CloseRegionProcedure 28230b65c0b592a41b759dc13b865232, server=ae3311ea5ccb,37917,1731708686516 in 1.9120 sec 2024-11-15T22:14:28,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=176, resume processing ppid=175 2024-11-15T22:14:28,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, ppid=175, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=28230b65c0b592a41b759dc13b865232, UNASSIGN in 1.9160 sec 2024-11-15T22:14:28,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-15T22:14:28,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9190 sec 2024-11-15T22:14:28,577 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708868576"}]},"ts":"1731708868576"} 2024-11-15T22:14:28,578 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T22:14:28,578 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T22:14:28,579 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9280 sec 2024-11-15T22:14:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-15T22:14:28,796 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:14:28,797 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T22:14:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:14:28,798 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=178, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:14:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-15T22:14:28,799 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=178, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:14:28,801 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,803 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A, FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B, FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C, FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/recovered.edits] 2024-11-15T22:14:28,805 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/69c5d54641ec490e96eb53412de6b65f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/69c5d54641ec490e96eb53412de6b65f 2024-11-15T22:14:28,806 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/fffe5ad793dd4b319156bd2cc232ad69 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/A/fffe5ad793dd4b319156bd2cc232ad69 2024-11-15T22:14:28,808 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/1436d4d5778849768299086356da122f to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/1436d4d5778849768299086356da122f 2024-11-15T22:14:28,808 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/e0720baf01854a5f9f6cd8a0976eacbd to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/B/e0720baf01854a5f9f6cd8a0976eacbd 2024-11-15T22:14:28,811 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/3bd820be359e44b7ae15bd8122c72ff0 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/3bd820be359e44b7ae15bd8122c72ff0 2024-11-15T22:14:28,814 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/d13f82b975aa4384ad745fba937f9431 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/C/d13f82b975aa4384ad745fba937f9431 2024-11-15T22:14:28,817 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/recovered.edits/466.seqid to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232/recovered.edits/466.seqid 2024-11-15T22:14:28,817 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/default/TestAcidGuarantees/28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,817 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T22:14:28,818 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T22:14:28,819 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T22:14:28,822 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115053181a62f2742c29182b16cf6a253e7_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115053181a62f2742c29182b16cf6a253e7_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,823 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111539efa8ad7801483eb02e24d422449178_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111539efa8ad7801483eb02e24d422449178_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,824 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153a7453b183f74fd9bfe4cc7dda32ff8d_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153a7453b183f74fd9bfe4cc7dda32ff8d_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,825 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153cfbf9a14ce6400fac169a8bb234a304_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153cfbf9a14ce6400fac169a8bb234a304_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,826 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153dbb42034a454eb7bccdd8632b3bb551_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153dbb42034a454eb7bccdd8632b3bb551_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,827 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115455cb90e008f462bb6be0568d15b8b88_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115455cb90e008f462bb6be0568d15b8b88_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,828 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154b706dff664a4901a159a03ce8754926_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154b706dff664a4901a159a03ce8754926_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,829 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155380ec7fcc414eb6a11c33e65ea8fc15_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155380ec7fcc414eb6a11c33e65ea8fc15_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,830 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155715f031355d4f309f840c3f0754fa29_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155715f031355d4f309f840c3f0754fa29_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,830 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115676fc6b0f3874146bf95f5867053944e_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115676fc6b0f3874146bf95f5867053944e_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,831 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157fc3b5f021874c8bb1ae7d12e0973510_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157fc3b5f021874c8bb1ae7d12e0973510_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,832 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111590a91d28f4d246639804e7c65ad6ad53_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111590a91d28f4d246639804e7c65ad6ad53_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,834 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111594f69ed53f8a441480250cf1c26fab0b_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111594f69ed53f8a441480250cf1c26fab0b_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,835 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159af0693a7d704cc4b3a122b72137b039_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159af0693a7d704cc4b3a122b72137b039_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,836 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159c6dcb40c7fe4dbbae0bc96f832926e5_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159c6dcb40c7fe4dbbae0bc96f832926e5_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,837 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159d94f8f4ffe449fabc0b14e20751a712_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159d94f8f4ffe449fabc0b14e20751a712_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,838 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115acbfa846a5c74cf8a5d810ff268f8231_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115acbfa846a5c74cf8a5d810ff268f8231_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,839 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b2db311533e1436691aced39d67ad990_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b2db311533e1436691aced39d67ad990_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,840 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d74d6b781a7f4823b40ec446485688a7_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d74d6b781a7f4823b40ec446485688a7_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,841 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d8de4441d2b64af285b5ca980914901d_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d8de4441d2b64af285b5ca980914901d_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,842 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e8216468fdfc4eb38f06a7af812c952b_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e8216468fdfc4eb38f06a7af812c952b_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,843 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ed2956b10a73466db0193b30efd6b5dc_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ed2956b10a73466db0193b30efd6b5dc_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,845 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f0d5a46c5bd845e6a68a54ac552c3f6b_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f0d5a46c5bd845e6a68a54ac552c3f6b_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,846 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f8258c94c8f242bca099827e34ba6f54_28230b65c0b592a41b759dc13b865232 to hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f8258c94c8f242bca099827e34ba6f54_28230b65c0b592a41b759dc13b865232 2024-11-15T22:14:28,846 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T22:14:28,848 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=178, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:14:28,851 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T22:14:28,853 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T22:14:28,854 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=178, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:14:28,854 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T22:14:28,854 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731708868854"}]},"ts":"9223372036854775807"} 2024-11-15T22:14:28,856 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T22:14:28,856 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 28230b65c0b592a41b759dc13b865232, NAME => 'TestAcidGuarantees,,1731708837945.28230b65c0b592a41b759dc13b865232.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T22:14:28,856 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T22:14:28,856 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731708868856"}]},"ts":"9223372036854775807"} 2024-11-15T22:14:28,857 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T22:14:28,858 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=178, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T22:14:28,859 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 61 msec 2024-11-15T22:14:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-15T22:14:28,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T22:14:28,910 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T22:14:28,917 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=242 (was 241) - Thread LEAK? -, OpenFileDescriptor=459 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=515 (was 604), ProcessCount=11 (was 11), AvailableMemoryMB=2399 (was 2227) - AvailableMemoryMB LEAK? - 2024-11-15T22:14:28,917 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-15T22:14:28,917 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T22:14:28,917 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:14:28,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:28,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:28,917 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:14:28,917 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-15T22:14:28,917 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1258311727, stopped=false 2024-11-15T22:14:28,918 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ae3311ea5ccb,37303,1731708685620 2024-11-15T22:14:28,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T22:14:28,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37917-0x100f8747e0b0001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T22:14:28,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:28,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37917-0x100f8747e0b0001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:28,920 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:28,920 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T22:14:28,920 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:14:28,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:28,921 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37917-0x100f8747e0b0001, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T22:14:28,921 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ae3311ea5ccb,37917,1731708686516' ***** 2024-11-15T22:14:28,921 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T22:14:28,922 INFO [RS:0;ae3311ea5ccb:37917 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T22:14:28,922 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T22:14:28,922 INFO [RS:0;ae3311ea5ccb:37917 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T22:14:28,922 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T22:14:28,922 INFO [RS:0;ae3311ea5ccb:37917 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T22:14:28,922 INFO [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(959): stopping server ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:28,922 INFO [RS:0;ae3311ea5ccb:37917 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T22:14:28,923 INFO [RS:0;ae3311ea5ccb:37917 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ae3311ea5ccb:37917. 2024-11-15T22:14:28,923 DEBUG [RS:0;ae3311ea5ccb:37917 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:14:28,923 DEBUG [RS:0;ae3311ea5ccb:37917 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:28,923 INFO [RS:0;ae3311ea5ccb:37917 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T22:14:28,923 INFO [RS:0;ae3311ea5ccb:37917 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T22:14:28,923 INFO [RS:0;ae3311ea5ccb:37917 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T22:14:28,923 INFO [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T22:14:28,924 INFO [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-15T22:14:28,924 DEBUG [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-15T22:14:28,924 DEBUG [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T22:14:28,924 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T22:14:28,925 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T22:14:28,925 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T22:14:28,925 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T22:14:28,925 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T22:14:28,925 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-15T22:14:28,933 INFO [regionserver/ae3311ea5ccb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T22:14:28,958 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/.tmp/info/e76ed0cfe83d42bc9ca8f9b615ab71b0 is 95, key is TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414./info:/1731708718150/DeleteFamily/seqid=0 2024-11-15T22:14:28,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742514_1690 (size=6514) 2024-11-15T22:14:29,125 DEBUG [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T22:14:29,325 DEBUG [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T22:14:29,362 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/.tmp/info/e76ed0cfe83d42bc9ca8f9b615ab71b0 2024-11-15T22:14:29,382 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/.tmp/ns/9148b5c6af424ef491be86a74ceaddda is 93, key is TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414./ns:/1731708718101/DeleteFamily/seqid=0 2024-11-15T22:14:29,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742515_1691 (size=5906) 2024-11-15T22:14:29,525 DEBUG [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T22:14:29,725 DEBUG [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T22:14:29,729 INFO [regionserver/ae3311ea5ccb:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-15T22:14:29,729 INFO [regionserver/ae3311ea5ccb:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-15T22:14:29,786 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/.tmp/ns/9148b5c6af424ef491be86a74ceaddda 2024-11-15T22:14:29,805 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/.tmp/rep_barrier/ce00202c88684d47b8ba647c043afb66 is 102, key is TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414./rep_barrier:/1731708718101/DeleteFamily/seqid=0 2024-11-15T22:14:29,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742516_1692 (size=6025) 2024-11-15T22:14:29,926 DEBUG [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T22:14:30,126 DEBUG [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T22:14:30,209 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/.tmp/rep_barrier/ce00202c88684d47b8ba647c043afb66 2024-11-15T22:14:30,232 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/.tmp/table/b3a259b04c0c4b1fb583e3b2dae51801 is 96, key is TestAcidGuarantees,,1731708689942.3220684e1e7cfe8432a04d7ee5eae414./table:/1731708718101/DeleteFamily/seqid=0 2024-11-15T22:14:30,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742517_1693 (size=5936) 2024-11-15T22:14:30,241 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/.tmp/table/b3a259b04c0c4b1fb583e3b2dae51801 2024-11-15T22:14:30,246 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/.tmp/info/e76ed0cfe83d42bc9ca8f9b615ab71b0 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/info/e76ed0cfe83d42bc9ca8f9b615ab71b0 2024-11-15T22:14:30,249 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/info/e76ed0cfe83d42bc9ca8f9b615ab71b0, entries=12, sequenceid=90, filesize=6.4 K 2024-11-15T22:14:30,250 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/.tmp/ns/9148b5c6af424ef491be86a74ceaddda as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/ns/9148b5c6af424ef491be86a74ceaddda 2024-11-15T22:14:30,253 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/ns/9148b5c6af424ef491be86a74ceaddda, entries=8, sequenceid=90, filesize=5.8 K 2024-11-15T22:14:30,253 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/.tmp/rep_barrier/ce00202c88684d47b8ba647c043afb66 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/rep_barrier/ce00202c88684d47b8ba647c043afb66 2024-11-15T22:14:30,256 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/rep_barrier/ce00202c88684d47b8ba647c043afb66, entries=6, sequenceid=90, filesize=5.9 K 2024-11-15T22:14:30,256 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/.tmp/table/b3a259b04c0c4b1fb583e3b2dae51801 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/table/b3a259b04c0c4b1fb583e3b2dae51801 2024-11-15T22:14:30,259 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/table/b3a259b04c0c4b1fb583e3b2dae51801, entries=7, sequenceid=90, filesize=5.8 K 2024-11-15T22:14:30,260 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1335ms, sequenceid=90, compaction requested=false 2024-11-15T22:14:30,265 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-15T22:14:30,266 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T22:14:30,266 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T22:14:30,266 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731708868924Running coprocessor pre-close hooks at 1731708868924Disabling compacts and flushes for region at 1731708868924Disabling writes for close at 1731708868925 (+1 ms)Obtaining lock to block concurrent updates at 1731708868925Preparing flush snapshotting stores in 1588230740 at 1731708868925Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731708868925Flushing stores of hbase:meta,,1.1588230740 at 1731708868933 (+8 ms)Flushing 1588230740/info: creating writer at 1731708868934 (+1 ms)Flushing 1588230740/info: appending metadata at 1731708868958 (+24 ms)Flushing 1588230740/info: closing flushed file at 1731708868958Flushing 1588230740/ns: creating writer at 1731708869367 (+409 ms)Flushing 1588230740/ns: appending metadata at 1731708869382 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731708869382Flushing 1588230740/rep_barrier: creating writer at 1731708869789 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731708869805 (+16 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731708869805Flushing 1588230740/table: creating writer at 1731708870213 (+408 ms)Flushing 1588230740/table: appending metadata at 1731708870232 (+19 ms)Flushing 1588230740/table: closing flushed file at 1731708870232Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1417f7ba: reopening flushed file at 1731708870245 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48af9039: reopening flushed file at 1731708870250 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ca51eca: reopening flushed file at 1731708870253 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5eeca621: reopening flushed file at 1731708870256 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1335ms, sequenceid=90, compaction requested=false at 1731708870260 (+4 ms)Writing region close event to WAL at 1731708870262 (+2 ms)Running coprocessor post-close hooks at 1731708870266 (+4 ms)Closed at 1731708870266 2024-11-15T22:14:30,266 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T22:14:30,326 INFO [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(976): stopping server ae3311ea5ccb,37917,1731708686516; all regions closed. 2024-11-15T22:14:30,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741834_1010 (size=25344) 2024-11-15T22:14:30,333 DEBUG [RS:0;ae3311ea5ccb:37917 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/oldWALs 2024-11-15T22:14:30,333 INFO [RS:0;ae3311ea5ccb:37917 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ae3311ea5ccb%2C37917%2C1731708686516.meta:.meta(num 1731708689361) 2024-11-15T22:14:30,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741832_1008 (size=15356250) 2024-11-15T22:14:30,337 DEBUG [RS:0;ae3311ea5ccb:37917 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/oldWALs 2024-11-15T22:14:30,337 INFO [RS:0;ae3311ea5ccb:37917 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ae3311ea5ccb%2C37917%2C1731708686516:(num 1731708688823) 2024-11-15T22:14:30,337 DEBUG [RS:0;ae3311ea5ccb:37917 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:30,337 INFO [RS:0;ae3311ea5ccb:37917 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T22:14:30,337 INFO [RS:0;ae3311ea5ccb:37917 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T22:14:30,337 INFO [RS:0;ae3311ea5ccb:37917 {}] hbase.ChoreService(370): Chore service for: regionserver/ae3311ea5ccb:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-15T22:14:30,338 INFO [RS:0;ae3311ea5ccb:37917 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T22:14:30,338 INFO [regionserver/ae3311ea5ccb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T22:14:30,338 INFO [RS:0;ae3311ea5ccb:37917 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37917 2024-11-15T22:14:30,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T22:14:30,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37917-0x100f8747e0b0001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ae3311ea5ccb,37917,1731708686516 2024-11-15T22:14:30,342 INFO [RS:0;ae3311ea5ccb:37917 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T22:14:30,345 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ae3311ea5ccb,37917,1731708686516] 2024-11-15T22:14:30,346 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ae3311ea5ccb,37917,1731708686516 already deleted, retry=false 2024-11-15T22:14:30,346 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ae3311ea5ccb,37917,1731708686516 expired; onlineServers=0 2024-11-15T22:14:30,347 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ae3311ea5ccb,37303,1731708685620' ***** 2024-11-15T22:14:30,347 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-15T22:14:30,347 INFO [M:0;ae3311ea5ccb:37303 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T22:14:30,347 INFO [M:0;ae3311ea5ccb:37303 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T22:14:30,347 DEBUG [M:0;ae3311ea5ccb:37303 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-15T22:14:30,347 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-15T22:14:30,347 DEBUG [M:0;ae3311ea5ccb:37303 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-15T22:14:30,347 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.large.0-1731708688488 {}] cleaner.HFileCleaner(306): Exit Thread[master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.large.0-1731708688488,5,FailOnTimeoutGroup] 2024-11-15T22:14:30,347 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.small.0-1731708688489 {}] cleaner.HFileCleaner(306): Exit Thread[master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.small.0-1731708688489,5,FailOnTimeoutGroup] 2024-11-15T22:14:30,347 INFO [M:0;ae3311ea5ccb:37303 {}] hbase.ChoreService(370): Chore service for: master/ae3311ea5ccb:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-15T22:14:30,347 INFO [M:0;ae3311ea5ccb:37303 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T22:14:30,347 DEBUG [M:0;ae3311ea5ccb:37303 {}] master.HMaster(1795): Stopping service threads 2024-11-15T22:14:30,347 INFO [M:0;ae3311ea5ccb:37303 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-15T22:14:30,348 INFO [M:0;ae3311ea5ccb:37303 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:30,348 ERROR [M:0;ae3311ea5ccb:37303 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-15T22:14:30,348 INFO [M:0;ae3311ea5ccb:37303 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:30,348 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-15T22:14:30,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-15T22:14:30,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:30,349 DEBUG [M:0;ae3311ea5ccb:37303 {}] zookeeper.ZKUtil(347): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-15T22:14:30,349 WARN [M:0;ae3311ea5ccb:37303 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-15T22:14:30,350 INFO [M:0;ae3311ea5ccb:37303 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/.lastflushedseqids 2024-11-15T22:14:30,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742518_1694 (size=72) 2024-11-15T22:14:30,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37917-0x100f8747e0b0001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T22:14:30,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37917-0x100f8747e0b0001, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T22:14:30,446 INFO [RS:0;ae3311ea5ccb:37917 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T22:14:30,446 INFO [RS:0;ae3311ea5ccb:37917 {}] regionserver.HRegionServer(1031): Exiting; stopping=ae3311ea5ccb,37917,1731708686516; zookeeper connection closed. 2024-11-15T22:14:30,446 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5c2de00a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5c2de00a 2024-11-15T22:14:30,447 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-15T22:14:30,760 INFO [M:0;ae3311ea5ccb:37303 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:30,760 INFO [M:0;ae3311ea5ccb:37303 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-15T22:14:30,760 DEBUG [M:0;ae3311ea5ccb:37303 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T22:14:30,760 INFO [M:0;ae3311ea5ccb:37303 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:14:30,761 DEBUG [M:0;ae3311ea5ccb:37303 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:14:30,761 DEBUG [M:0;ae3311ea5ccb:37303 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T22:14:30,761 DEBUG [M:0;ae3311ea5ccb:37303 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:14:30,761 INFO [M:0;ae3311ea5ccb:37303 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=775.41 KB heapSize=954.80 KB 2024-11-15T22:14:30,777 DEBUG [M:0;ae3311ea5ccb:37303 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/874c4575de48490ba9e362f5271d6ed6 is 82, key is hbase:meta,,1/info:regioninfo/1731708689453/Put/seqid=0 2024-11-15T22:14:30,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742519_1695 (size=5672) 2024-11-15T22:14:31,180 INFO [M:0;ae3311ea5ccb:37303 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2213 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/874c4575de48490ba9e362f5271d6ed6 2024-11-15T22:14:31,201 DEBUG [M:0;ae3311ea5ccb:37303 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a23e312562bb40a88d5735a382200ac4 is 2275, key is \x00\x00\x00\x00\x00\x00\x00\x97/proc:d/1731708841010/Put/seqid=0 2024-11-15T22:14:31,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742520_1696 (size=45519) 2024-11-15T22:14:31,604 INFO [M:0;ae3311ea5ccb:37303 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=774.86 KB at sequenceid=2213 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a23e312562bb40a88d5735a382200ac4 2024-11-15T22:14:31,609 INFO [M:0;ae3311ea5ccb:37303 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a23e312562bb40a88d5735a382200ac4 2024-11-15T22:14:31,626 DEBUG [M:0;ae3311ea5ccb:37303 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ad9690d09c1d42b08572cd3fb4d692f7 is 69, key is ae3311ea5ccb,37917,1731708686516/rs:state/1731708688589/Put/seqid=0 2024-11-15T22:14:31,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073742521_1697 (size=5156) 2024-11-15T22:14:32,030 INFO [M:0;ae3311ea5ccb:37303 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2213 (bloomFilter=true), to=hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ad9690d09c1d42b08572cd3fb4d692f7 2024-11-15T22:14:32,033 DEBUG [M:0;ae3311ea5ccb:37303 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/874c4575de48490ba9e362f5271d6ed6 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/874c4575de48490ba9e362f5271d6ed6 2024-11-15T22:14:32,036 INFO [M:0;ae3311ea5ccb:37303 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/874c4575de48490ba9e362f5271d6ed6, entries=8, sequenceid=2213, filesize=5.5 K 2024-11-15T22:14:32,037 DEBUG [M:0;ae3311ea5ccb:37303 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a23e312562bb40a88d5735a382200ac4 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a23e312562bb40a88d5735a382200ac4 2024-11-15T22:14:32,040 INFO [M:0;ae3311ea5ccb:37303 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a23e312562bb40a88d5735a382200ac4 2024-11-15T22:14:32,040 INFO [M:0;ae3311ea5ccb:37303 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a23e312562bb40a88d5735a382200ac4, entries=178, sequenceid=2213, filesize=44.5 K 2024-11-15T22:14:32,040 DEBUG [M:0;ae3311ea5ccb:37303 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ad9690d09c1d42b08572cd3fb4d692f7 as hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ad9690d09c1d42b08572cd3fb4d692f7 2024-11-15T22:14:32,043 INFO [M:0;ae3311ea5ccb:37303 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35615/user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ad9690d09c1d42b08572cd3fb4d692f7, entries=1, sequenceid=2213, filesize=5.0 K 2024-11-15T22:14:32,044 INFO [M:0;ae3311ea5ccb:37303 {}] regionserver.HRegion(3140): Finished flush of dataSize ~775.41 KB/794022, heapSize ~954.51 KB/977416, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1283ms, sequenceid=2213, compaction requested=false 2024-11-15T22:14:32,045 INFO [M:0;ae3311ea5ccb:37303 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:14:32,045 DEBUG [M:0;ae3311ea5ccb:37303 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731708870760Disabling compacts and flushes for region at 1731708870760Disabling writes for close at 1731708870761 (+1 ms)Obtaining lock to block concurrent updates at 1731708870761Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731708870761Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=794022, getHeapSize=977656, getOffHeapSize=0, getCellsCount=2416 at 1731708870761Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731708870761Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731708870762 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731708870776 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731708870776Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731708871183 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731708871200 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731708871200Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731708871609 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731708871625 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731708871625Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@94b90c7: reopening flushed file at 1731708872033 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e2ef188: reopening flushed file at 1731708872036 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c55fa9c: reopening flushed file at 1731708872040 (+4 ms)Finished flush of dataSize ~775.41 KB/794022, heapSize ~954.51 KB/977416, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1283ms, sequenceid=2213, compaction requested=false at 1731708872044 (+4 ms)Writing region close event to WAL at 1731708872045 (+1 ms)Closed at 1731708872045 2024-11-15T22:14:32,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35039 is added to blk_1073741830_1006 (size=939098) 2024-11-15T22:14:32,048 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/3c679e7e-fe0b-cd64-4519-7939eeb8bf31/MasterData/WALs/ae3311ea5ccb,37303,1731708685620/ae3311ea5ccb%2C37303%2C1731708685620.1731708687883 not finished, retry = 0 2024-11-15T22:14:32,149 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T22:14:32,150 INFO [M:0;ae3311ea5ccb:37303 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-15T22:14:32,150 INFO [M:0;ae3311ea5ccb:37303 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37303 2024-11-15T22:14:32,150 INFO [M:0;ae3311ea5ccb:37303 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T22:14:32,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T22:14:32,252 INFO [M:0;ae3311ea5ccb:37303 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T22:14:32,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37303-0x100f8747e0b0000, quorum=127.0.0.1:56467, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T22:14:32,258 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1ebb9f9f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T22:14:32,262 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@bd2e4d1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T22:14:32,263 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T22:14:32,263 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3de8243e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T22:14:32,263 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@288b0180{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/hadoop.log.dir/,STOPPED} 2024-11-15T22:14:32,266 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-15T22:14:32,266 WARN [BP-8675891-172.17.0.3-1731708682292 heartbeating to localhost/127.0.0.1:35615 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-15T22:14:32,266 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-15T22:14:32,266 WARN [BP-8675891-172.17.0.3-1731708682292 heartbeating to localhost/127.0.0.1:35615 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-8675891-172.17.0.3-1731708682292 (Datanode Uuid 723faa33-b9ca-4c30-bbb3-1907a5afaa48) service to localhost/127.0.0.1:35615 2024-11-15T22:14:32,269 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/cluster_048f81ed-d9d5-ea35-fb33-ebaa9399bbca/data/data1/current/BP-8675891-172.17.0.3-1731708682292 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T22:14:32,269 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/cluster_048f81ed-d9d5-ea35-fb33-ebaa9399bbca/data/data2/current/BP-8675891-172.17.0.3-1731708682292 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T22:14:32,269 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-15T22:14:32,279 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@322e478f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T22:14:32,280 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T22:14:32,280 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T22:14:32,280 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T22:14:32,280 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/59f13eff-6dc8-6832-d87f-d89c987bf8cf/hadoop.log.dir/,STOPPED} 2024-11-15T22:14:32,297 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-15T22:14:32,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down